def test_input_data_required_must_be_true_or_false(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1}',
'version': '1.0',
'input_data': [{
'name': 'param-1',
'type': 'file',
'required': True,
}]
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
definition['input_data'][0]['required'] = False
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
definition['input_data'][0]['required'] = 'some_string'
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_no_workspace_needed(self):
"""Tests calling JobInterface.validate_connection() without a workspace, but none is needed."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.1',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain'],
'partial': True
}],
'output_data': [],
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, True)
# No exception is success
job_interface.validate_connection(job_conn)
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_isdir):
def new_retrieve(arg1):
return {
'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'],
}
mock_retrieve_call.side_effect = new_retrieve
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${files1}'
job_interface_dict['input_data'] = [{
'name': 'files1',
'type': 'files',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'files1',
'file_ids': [1, 2, 3],
})
job_data_dict['output_data'].append({
'name': 'files1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1')
self.assertEqual(job_command_arguments, expected_command_arguments,
'expected a different command from pre_steps')
def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_file',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/some/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_file': ('/some/path/foo.txt', None),
}, job_exe)
def test_successful(self):
"""Tests calling JobInterface.validate_connection() successfully."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.0',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain']
}],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, False)
job_conn.add_workspace()
# No exception is success
job_interface.validate_connection(job_conn)
def test_invalid_output_files(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_files',
'paths': ['/some/path/foo.txt', '/other/path/foo.txt'],
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = False
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
self.assertRaises(InvalidResultsManifest, job_interface.perform_post_steps, job_exe, job_data, fake_stdout)
def test_required_workspace_missing(self):
"""Tests calling JobInterface.validate_connection() when a required workspace is missing"""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.1',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain'],
'partial': True
}],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, True)
self.assertRaises(InvalidConnection, job_interface.validate_connection, job_conn)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_isdir):
job_exe_id = 1
def new_retrieve(arg1):
return {
'file1_out': [input_file_path],
}
input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt')
mock_retrieve_call.side_effect = new_retrieve
mock_get_one_file.side_effect = lambda (arg1): input_file_path
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${file1}'
job_interface_dict['input_data'] = [{
'name': 'file1',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'file1',
'file_id': self.file.id,
})
job_data_dict['output_data'].append({
'name': 'file1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def get_job_interface(self):
"""Returns the interface for this queued job
:returns: The job interface
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return JobInterface(self.interface, do_validate=False)
def job_type_get_job_interface(self):
"""Returns the interface for running jobs of this type
:returns: The job interface for this type
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return JobInterface(self.interface)
def job_get_job_interface(self):
"""Returns the interface for this job
:returns: The interface for this job
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return JobInterface(self.job_type_rev.interface)
def get_job_types(recipe_types=None,
job_type_ids=None,
job_type_names=None,
job_type_categories=None):
"""Exports all the job types in the system based on the given filters.
:param recipe_types: Only include job types that are referenced by the given recipe types.
:type recipe_types: list[:class:`recipe.models.RecipeType`]
:param job_type_ids: A list of unique job type identifiers to include.
:type job_type_ids: list[str]
:param job_type_names: A list of job type system names to include.
:type job_type_names: list[str]
:param job_type_categories: A list of job type category names to include.
:type job_type_categories: list[str]
:returns: A list of matching job types.
:rtype: list[:class:`job.models.JobType`]
"""
# Build a set of job type keys referenced by the recipe types
job_type_keys = set()
if recipe_types and not (job_type_ids or job_type_names
or job_type_categories):
for recipe_type in recipe_types:
job_type_keys.update(
recipe_type.get_recipe_definition().get_job_type_keys())
if not job_type_keys:
return []
# System job types should never be exported
job_types = JobType.objects.exclude(
category='system').select_related('trigger_rule')
# Filter by the referenced job type keys
job_type_filters = []
for job_type_key in job_type_keys:
job_type_filter = Q(name=job_type_key[0], version=job_type_key[1])
job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter
if job_type_filters:
job_types = job_types.filter(job_type_filters)
# Filter by additional passed arguments
if job_type_ids:
job_types = job_types.filter(id__in=job_type_ids)
if job_type_names:
job_types = job_types.filter(name__in=job_type_names)
if job_type_categories:
job_types = job_types.filter(category__in=job_type_categories)
# Scrub configuration for secrets
for job_type in job_types:
if job_type.configuration:
configuration = JobConfigurationV2(job_type.configuration)
interface = JobInterface(job_type.manifest)
configuration.validate(interface.get_dict())
job_type.configuration = configuration.get_dict()
return job_types
def test_output_files_with_geo_metadata(self, mock_loads, mock_open,
mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
geo_metadata = {
'data_started': '2015-05-15T10:34:12Z',
'data_ended': '2015-05-15T10:36:12Z',
'geo_json': {
'type':
'Polygon',
'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0],
[1.0, 20.0], [1.0, 10.0]]],
}
}
results_manifest = {
'version':
'1.1',
'output_data': [{
'name':
'output_files',
'files': [{
'path': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}, {
'path': '/other/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{
'output_files': [
('/some/path/foo.txt', None, geo_metadata),
('/other/path/foo.txt', None, geo_metadata),
]
}, job_exe)
def test_interface_must_have_command(self):
definition = {
'version': '1.0',
}
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_minimal_input_validation(self):
definition = {
'command': 'test-command',
'command_arguments': 'some_argument',
'version': '1.0',
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('A valid definition should not raise an Exception')
def test_bad_input_name(self):
"""Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a bad input name"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))
interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "different_input_name", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
job_interface = JobInterface(json.loads(interface_json_str))
self.assertRaises(InvalidConnection, rule_config.validate_trigger_for_job, job_interface)
def test_output_name_appropriate(self):
good_names = [
'foo', 'bar', 'baz', 'a file with spaces', 'file_with_underscores'
]
bad_names = [
'ca$h_money', 'do|do_not', 'try!=found',
'this_file_is_over_255_characters_long_12345678901234567890123456789012345678901234567890123456789012345678'
'9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234'
'56789012345678901234567890123456789012345678901234567890!'
]
definition = {
'command': 'test-command',
'command_arguments': '',
'version': '1.0',
'input_data': [{
'name': 'foo',
'type': 'file',
'required': True,
}],
'output_data': [{
'name': 'some_output',
'type': 'file',
}]
}
for output_name in good_names:
definition['output_data'][0]['name'] = output_name
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail(
'Unable to parse a good interface definition with output name: %s'
% output_name)
for output_name in bad_names:
definition['output_data'][0]['name'] = output_name
try:
JobInterface(definition)
self.fail(
'job interface with a bad output name (%s) was able to get past validation'
% output_name)
except InvalidInterfaceDefinition:
pass
def test_simple_case(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_populate_mounts(self):
"""Tests the addition of mount volumes to the configuration."""
exe_config = ExecutionConfiguration()
config_dict = {
'version': '2.0',
'mounts': {
'mount_1': {
'type': 'host',
'host_path': '/host/path'
},
'mount_2': {
'type': 'volume',
'driver': 'x-driver',
'driver_opts': {
'foo': 'bar'
}
}
}
}
interface_dict = {
'version':
'1.4',
'command':
'the cmd',
'command_arguments':
'foo',
'mounts': [{
'name': 'mount_1',
'path': '/mount_1',
'mode': 'ro'
}, {
'name': 'mount_2',
'path': '/mount_2',
'mode': 'rw'
}]
}
job_exe = MagicMock()
job_exe.get_job_configuration.return_value = JobConfiguration(
config_dict)
job_exe.get_job_interface.return_value = JobInterface(interface_dict)
job_exe.get_cluster_id.return_value = 'scale_1234'
exe_config.populate_mounts(job_exe)
docker_params = exe_config.get_job_task_docker_params()
self.assertEqual(docker_params[0].flag, 'volume')
self.assertEqual(docker_params[0].value, '/host/path:/mount_1:ro')
self.assertEqual(docker_params[1].flag, 'volume')
mount_2 = '$(docker volume create --name scale_1234_mount_mount_2 --driver x-driver --opt foo=bar):/mount_2:rw'
self.assertEqual(docker_params[1].value, mount_2)
def test_command_param_will_fail_without_input(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1}',
'version': '1.0',
}
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_manifest_overrides_stdout(self, mock_loads, mock_open,
mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}, {
'name': 'output_file_2',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/new/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = """
This text is supposed to mimic the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
"""
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{
'output_file': ('/new/path/foo.txt', None),
'output_file_2': ('/path/to/foo_2.txt', None),
}, job_exe)
def test_media_type_warning(self):
"""Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a warning for a mis-matched media type"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))
interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
job_interface = JobInterface(json.loads(interface_json_str))
warnings = rule_config.validate_trigger_for_job(job_interface)
self.assertEqual(len(warnings), 1)
def test_successful(self):
"""Tests calling ParseTriggerRuleConfiguration.validate_trigger_for_job() successfully with no warnings"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(rule_json_str))
interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
job_interface = JobInterface(json.loads(interface_json_str))
warnings = rule_config.validate_trigger_for_job(job_interface)
self.assertListEqual(warnings, [])
def test_output_dir_in_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${job_output_dir}'
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_output_dir = SCALE_JOB_EXE_OUTPUT_PATH
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_command_param_will_pass_with_input(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1}',
'version': '1.0',
'input_data': [{
'name': 'param-1',
'type': 'file',
}]
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
def test_command_string_allows_special_formats(self):
definition = {
'command': 'test-command',
'command_arguments': '${-f :param-1}',
'version': '1.0',
'input_data': [{
'name': 'param-1',
'type': 'file',
}]
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
def test_input_data_must_have_a_type(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1}',
'version': '1.0',
'input_data': [{
'name': 'param-1',
}]
}
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_interface_with_share_resource_works(self):
definition = {
'command': 'test-command',
'command_arguments': '',
'version': '1.0',
'shared_resources': [{
'name': 'resource-1',
'type': 'db-connection',
}]
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
def create(interface_dict, do_validate=True):
"""Instantiate an instance of the JobInterface based on inferred type
:param interface_dict: deserialized JSON interface
:type interface_dict: dict
:param do_validate: whether schema validation should be applied
:type do_validate: bool
:return: instance of the job interface appropriate for input data
:rtype: :class:`job.configuration.interface.job_interface.JobInterface` or
:class:`job.seed.manifest.SeedManifest`
"""
if JobInterfaceSunset.is_seed_dict(interface_dict):
return SeedManifest(interface_dict, do_validate=do_validate)
else:
return JobInterface(interface_dict, do_validate=do_validate)
def test_command_string_special_formats_should_have_dollar_sign(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1:-f param-1}',
'version': '1.0',
'input_data': [{
'name': 'param-1',
'type': 'file',
}]
}
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
