def test_input_data_required_must_be_true_or_false(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1}',
'version': '1.0',
'input_data': [{
'name': 'param-1',
'type': 'file',
'required': True,
}]
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
definition['input_data'][0]['required'] = False
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('Valid definition raised a validation exception')
definition['input_data'][0]['required'] = 'some_string'
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_no_workspace_needed(self):
"""Tests calling JobInterface.validate_connection() without a workspace, but none is needed."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.1',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain'],
'partial': True
}],
'output_data': [],
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, True)
# No exception is success
job_interface.validate_connection(job_conn)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file):
job_exe_id = 1
def new_retrieve(arg1):
return {
'file1_out': [input_file_path],
}
input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt')
mock_retrieve_call.side_effect = new_retrieve
mock_get_one_file.side_effect = lambda(arg1): input_file_path
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${file1}'
job_interface_dict['input_data'] = [{
'name': 'file1',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'file1',
'file_id': self.file.id,
})
job_data_dict['output_data'].append({
'name': 'file1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_isdir):
def new_retrieve(arg1):
return {
'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'],
}
mock_retrieve_call.side_effect = new_retrieve
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${files1}'
job_interface_dict['input_data'] = [{
'name': 'files1',
'type': 'files',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'files1',
'file_ids': [1, 2, 3],
})
job_data_dict['output_data'].append({
'name': 'files1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1')
self.assertEqual(job_command_arguments, expected_command_arguments,
'expected a different command from pre_steps')
def test_successful(self):
"""Tests calling JobInterface.validate_connection() successfully."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.0',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain']
}],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, False)
job_conn.add_workspace()
# No exception is success
job_interface.validate_connection(job_conn)
def test_output_files(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name':'output_files',
'paths': ['/some/path/foo.txt', '/other/path/foo.txt'],
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_files': [
('/some/path/foo.txt', None),
('/other/path/foo.txt', None),
]
}, job_exe)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_setup_upload):
job_work_dir = "/test"
job_exe_id = 1
job_input_dir = get_job_exe_input_data_dir(job_exe_id)
job_output_dir = os.path.join(job_work_dir, u'outputs')
def new_retrieve(arg1, arg2, arg3):
return {u'file1_out': [input_file_path]}
input_file_path = os.path.join(job_input_dir, 'file1', 'foo.txt')
mock_retrieve_call.side_effect = new_retrieve
mock_get_one_file.side_effect = lambda(arg1): input_file_path
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict[u'command_arguments'] = u'${file1}'
job_interface_dict[u'input_data'] = [{u'name' : u'file1', u'type' : u'file', 'required' : True}]
job_data_dict[u'input_data'].append({u'name': u'file1', u'file_id': 1})
job_data_dict[u'output_data'].append({u'name': u'file1_out', u'workspace_id': self.workspace.id})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_interface.perform_pre_steps(job_data, job_environment, job_exe_id)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, input_file_path, u'expected a different command from pre_steps')
mock_setup_upload.assert_called_once_with(get_job_exe_output_data_dir(job_exe_id), get_job_exe_output_work_dir(job_exe_id), self.workspace)
def test_extra_products_are_fine(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}]
job_interface_dict[u'output_data'] = [{"name":"output_file", "type":"file", "required": True}]
job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1})
results_manifest={"version" : "1.0", "files" : [{"name":"output_file", "path":"/new/path/foo.txt"}]}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = '''
This text is supposed to mimick the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
'''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({u'output_file': (u'/new/path/foo.txt', None)}, job_exe)
def test_successful(self):
"""Tests calling JobInterface.validate_connection() successfully."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.0',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain']
}],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, False)
job_conn.add_workspace()
# No exception is success
job_interface.validate_connection(job_conn)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir, mock_get_one_file, mock_isdir):
job_exe_id = 1
def new_retrieve(arg1):
return {
'file1_out': [input_file_path],
}
input_file_path = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'file1', 'foo.txt')
mock_retrieve_call.side_effect = new_retrieve
mock_get_one_file.side_effect = lambda (arg1): input_file_path
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${file1}'
job_interface_dict['input_data'] = [{
'name': 'file1',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'file1',
'file_id': self.file.id,
})
job_data_dict['output_data'].append({
'name': 'file1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, input_file_path, 'expected a different command from pre_steps')
def test_no_workspace_needed(self):
'''Tests calling JobInterface.validate_connection() without a workspace, but none is needed.'''
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.0',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [],
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False)
# No exception is success
job_interface.validate_connection(job_conn)
def test_output_file(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_file',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/some/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_file': ('/some/path/foo.txt', None),
}, job_exe)
def test_files_in_command(self, mock_retrieve_call, mock_os_mkdir):
def new_retrieve(arg1):
return {
'files1_out': ['/test/file1/foo.txt', '/test/file1/bar.txt'],
}
mock_retrieve_call.side_effect = new_retrieve
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${files1}'
job_interface_dict['input_data'] = [{
'name': 'files1',
'type': 'files',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'files1',
'file_ids': [1, 2, 3],
})
job_data_dict['output_data'].append({
'name': 'files1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
expected_command_arguments = os.path.join(SCALE_JOB_EXE_INPUT_PATH, 'files1')
self.assertEqual(job_command_arguments, expected_command_arguments,
'expected a different command from pre_steps')
def test_output_files_with_geo_metadata(self, mock_loads, mock_open,
mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
geo_metadata = {
'data_started': '2015-05-15T10:34:12Z',
'data_ended': '2015-05-15T10:36:12Z',
'geo_json': {
'type':
'Polygon',
'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0],
[1.0, 20.0], [1.0, 10.0]]],
}
}
results_manifest = {
'version':
'1.1',
'output_data': [{
'name':
'output_files',
'files': [{
'path': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}, {
'path': '/other/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{
'output_files': [
('/some/path/foo.txt', None, geo_metadata),
('/other/path/foo.txt', None, geo_metadata),
]
}, job_exe)
def get_job_types(recipe_types=None,
job_type_ids=None,
job_type_names=None,
job_type_categories=None):
"""Exports all the job types in the system based on the given filters.
:param recipe_types: Only include job types that are referenced by the given recipe types.
:type recipe_types: list[:class:`recipe.models.RecipeType`]
:param job_type_ids: A list of unique job type identifiers to include.
:type job_type_ids: list[str]
:param job_type_names: A list of job type system names to include.
:type job_type_names: list[str]
:param job_type_categories: A list of job type category names to include.
:type job_type_categories: list[str]
:returns: A list of matching job types.
:rtype: list[:class:`job.models.JobType`]
"""
# Build a set of job type keys referenced by the recipe types
job_type_keys = set()
if recipe_types and not (job_type_ids or job_type_names
or job_type_categories):
for recipe_type in recipe_types:
job_type_keys.update(
recipe_type.get_recipe_definition().get_job_type_keys())
if not job_type_keys:
return []
# System job types should never be exported
job_types = JobType.objects.exclude(
category='system').select_related('trigger_rule')
# Filter by the referenced job type keys
job_type_filters = []
for job_type_key in job_type_keys:
job_type_filter = Q(name=job_type_key[0], version=job_type_key[1])
job_type_filters = job_type_filters | job_type_filter if job_type_filters else job_type_filter
if job_type_filters:
job_types = job_types.filter(job_type_filters)
# Filter by additional passed arguments
if job_type_ids:
job_types = job_types.filter(id__in=job_type_ids)
if job_type_names:
job_types = job_types.filter(name__in=job_type_names)
if job_type_categories:
job_types = job_types.filter(category__in=job_type_categories)
# Scrub configuration for secrets
for job_type in job_types:
if job_type.configuration:
configuration = JobConfigurationV2(job_type.configuration)
interface = JobInterface(job_type.manifest)
configuration.validate(interface.get_dict())
job_type.configuration = configuration.get_dict()
return job_types
def test_simple_case(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment, 1)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_simple_case(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_manifest_overrides_stdout(self, mock_loads, mock_open,
mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}, {
'name': 'output_file_2',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/new/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = """
This text is supposed to mimic the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
"""
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{
'output_file': ('/new/path/foo.txt', None),
'output_file_2': ('/path/to/foo_2.txt', None),
}, job_exe)
def test_output_dir_in_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${job_output_dir}'
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_output_dir = SCALE_JOB_EXE_OUTPUT_PATH
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_output_dir_in_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${job_output_dir}'
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_exe_id = 1
job_output_dir = file_system.get_job_exe_output_data_dir(job_exe_id)
job_interface.perform_pre_steps(job_data, job_environment, 1)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, job_output_dir, 'expected a different command from pre_steps')
def test_manifest_overrides_stdout(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_interface_dict['output_data'] = [{
'name': 'output_file',
'type': 'file',
'required': True,
}, {
'name': 'output_file_2',
'type':'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_file',
'path': '/new/path/foo.txt',
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = """
This text is supposed to mimic the output
of a program we should see artifacts registered with
the format: ARTIFACT:<input-name>:path, but it needs be at the beginning of a line
so the example above won't match, but this will
ARTIFACT:output_file:/path/to/foo.txt
We should also be able to have text after the artifact and multiple artifacts.
ARTIFACT:output_file_2:/path/to/foo_2.txt
"""
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_file': ('/new/path/foo.txt', None),
'output_file_2': ('/path/to/foo_2.txt', None),
}, job_exe)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
geo_metadata = {
'data_started': '2015-05-15T10:34:12Z',
'data_ended': '2015-05-15T10:36:12Z',
'geo_json': {
'type': 'Polygon',
'coordinates': [[[1.0, 10.0], [2.0, 10.0], [2.0, 20.0], [1.0, 20.0], [1.0, 10.0]]],
}
}
results_manifest = {
'version': '1.1',
'output_data': [{
'name': 'output_files',
'files': [{
'path': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}, {
'path': '/other/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({
'output_files': [
('/some/path/foo.txt', None, geo_metadata),
('/other/path/foo.txt', None, geo_metadata),
]
}, job_exe)
def test_successful(self):
'''Tests calling JobInterface.add_output_to_connection() successfully.'''
job_interface_dict = {
u'command': u'simple-command',
u'command_arguments': u'',
u'version': u'1.0',
u'input_data': [],
u'output_data': [{u'name': u'Output 1', u'type': u'file'}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = MagicMock()
job_interface.add_output_to_connection(u'Output 1', job_conn, u'Input 1')
job_conn.add_input_file.assert_called_with(u'Input 1', False, [], False)
def test_invalid_output_files(self, mock_loads, mock_open, mock_exists, mock_isfile):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['output_data'] = [{
'name': 'output_files',
'type': 'files',
'required': True,
}]
job_data_dict['output_data'].append({
'name': 'output_files',
'workspace_id': self.workspace.id,
})
results_manifest = {
'version': '1.0',
'files': [{
'name': 'output_files',
'paths': ['/some/path/foo.txt', '/other/path/foo.txt'],
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
mock_isfile.return_value = False
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
self.assertRaises(InvalidResultsManifest, job_interface.perform_post_steps, job_exe, job_data, fake_stdout)
def test_required_workspace_missing(self):
"""Tests calling JobInterface.validate_connection() when a required workspace is missing"""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.1',
'input_data': [{
'name': 'Input 1',
'type': 'property',
}, {
'name': 'Input 2',
'type': 'file',
'media_types': ['text/plain'],
'partial': True
}],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property('Input 1')
job_conn.add_input_file('Input 2', False, ['text/plain'], False, True)
self.assertRaises(InvalidConnection, job_interface.validate_connection, job_conn)
def test_successful(self, mock_get_dict):
"""Tests calling JobInterface.update() successfully."""
mock_get_dict.return_value = self.job_interface_dict
job_interface = JobInterface.convert_interface(self.job_interface_dict)
self.assertEqual(job_interface['version'], '1.1')
self.assertIn('partial', job_interface['input_data'][0])
self.assertFalse(job_interface['input_data'][0]['partial'])
def test_successful(self, mock_get_dict):
"""Tests calling JobInterface.update() successfully."""
mock_get_dict.return_value = self.job_interface_dict
job_interface = JobInterface.convert_interface(self.job_interface_dict)
self.assertEqual(job_interface['version'], '1.1')
self.assertIn('partial', job_interface['input_data'][0])
self.assertFalse(job_interface['input_data'][0]['partial'])
def test_complex_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict[u'command_arguments'] = u'${-f :prop1}'
job_interface_dict[u'input_data'] = [{u'name' : u'prop1', u'type' : u'property', 'required' : True}]
job_data_dict[u'input_data'].append({u'name': u'prop1', u'value': u'property-value'})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_work_dir = "/test"
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment, 1)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, u'-f property-value', u'expected a different command from pre_steps')
def job_type_get_job_interface(self):
"""Returns the interface for running jobs of this type
:returns: The job interface for this type
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return JobInterface(self.interface)
def get_job_interface(self):
"""Returns the interface for this queued job
:returns: The job interface
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return JobInterface(self.interface, do_validate=False)
def job_get_job_interface(self):
"""Returns the interface for this job
:returns: The interface for this job
:rtype: :class:`job.configuration.interface.job_interface.JobInterface`
"""
return JobInterface(self.job_type_rev.interface)
def test_output_files(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}]
job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1})
results_manifest={"version" : "1.0", "files" : [{"name":"output_files", "paths": ["/some/path/foo.txt", "/other/path/foo.txt"]}]}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = u''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with({u'output_files': [(u'/some/path/foo.txt', None),(u'/other/path/foo.txt', None)]}, job_exe)
def test_successful(self):
"""Tests calling JobInterface.add_output_to_connection() successfully."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.0',
'input_data': [],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = MagicMock()
job_interface.add_output_to_connection('Output 1', job_conn, 'Input 1')
job_conn.add_input_file.assert_called_with('Input 1', False, [], False, False)
def test_interface_must_have_command(self):
definition = {
'version': '1.0',
}
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_absent_optional_files_in_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${input_files}'
job_interface_dict['input_data'] = [{
'name': 'input_files',
'type': 'files',
'required': False,
}]
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, '', 'expected a different command from pre_steps')
def test_successful(self):
"""Tests calling JobInterface.add_output_to_connection() successfully."""
job_interface_dict = {
'command': 'simple-command',
'command_arguments': '',
'version': '1.0',
'input_data': [],
'output_data': [{
'name': 'Output 1',
'type': 'file',
}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = MagicMock()
job_interface.add_output_to_connection('Output 1', job_conn, 'Input 1')
job_conn.add_input_file.assert_called_with('Input 1', False, [], False, False)
def test_output_files_with_geo_metadata(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'output_data'] = [{"name":"output_files", "type":"files", "required": True}]
job_data_dict[u'output_data'].append({"name":"output_files", "workspace_id":1})
geo_metadata = {
"data_started": "2015-05-15T10:34:12Z",
"data_ended": "2015-05-15T10:36:12Z",
"geo_json": {
"type": "Polygon",
"coordinates": [[[ 1.0, 10.0 ], [ 2.0, 10.0 ], [ 2.0, 20.0 ],[ 1.0, 20.0 ], [ 1.0, 10.0 ]]]
}
}
results_manifest = {
"version": "1.1",
"output_data": [
{
"name" : "output_files",
"files": [
{
"path" : "/some/path/foo.txt",
"geo_metadata": geo_metadata
},
{
"path" : "/other/path/foo.txt",
"geo_metadata": geo_metadata
}
]
}
]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = u''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.store_output_data_files.assert_called_with(
{u'output_files': [(u'/some/path/foo.txt', None, geo_metadata),(u'/other/path/foo.txt', None, geo_metadata)]}, job_exe)
def test_absent_required_files_in_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${input_files}'
job_interface_dict['input_data'] = [{
'name': 'input_files',
'type': 'files',
'required': True,
}]
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
self.assertRaises(InvalidData, lambda: job_interface.fully_populate_command_argument(job_data,
job_environment,
job_exe_id))
def test_bad_input_name(self):
"""Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a bad input name"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))
interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "different_input_name", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
job_interface = JobInterface(json.loads(interface_json_str))
self.assertRaises(InvalidConnection, rule_config.validate_trigger_for_job, job_interface)
def test_file_in_command(self, mock_retrieve_call, mock_os_mkdir,
mock_get_one_file, mock_setup_upload):
job_exe_id = 1
job_input_dir = file_system.get_job_exe_input_data_dir(job_exe_id)
def new_retrieve(arg1, arg2, arg3):
return {
'file1_out': [input_file_path],
}
input_file_path = os.path.join(job_input_dir, 'file1', 'foo.txt')
mock_retrieve_call.side_effect = new_retrieve
mock_get_one_file.side_effect = lambda (arg1): input_file_path
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env(
)
job_interface_dict['command_arguments'] = '${file1}'
job_interface_dict['input_data'] = [{
'name': 'file1',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'file1',
'file_id': self.file.id,
})
job_data_dict['output_data'].append({
'name': 'file1_out',
'workspace_id': self.workspace.id,
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_interface.perform_pre_steps(job_data, job_environment, job_exe_id)
job_command_arguments = job_interface.fully_populate_command_argument(
job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, input_file_path,
'expected a different command from pre_steps')
mock_setup_upload.assert_called_once_with(
file_system.get_job_exe_output_data_dir(job_exe_id),
file_system.get_job_exe_output_work_dir(job_exe_id),
self.workspace)
def test_minimal_input_validation(self):
definition = {
'command': 'test-command',
'command_arguments': 'some_argument',
'version': '1.0',
}
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail('A valid definition should not raise an Exception')
def test_output_name_appropriate(self):
good_names = [
'foo', 'bar', 'baz', 'a file with spaces', 'file_with_underscores'
]
bad_names = [
'ca$h_money', 'do|do_not', 'try!=found',
'this_file_is_over_255_characters_long_12345678901234567890123456789012345678901234567890123456789012345678'
'9012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234'
'56789012345678901234567890123456789012345678901234567890!'
]
definition = {
'command': 'test-command',
'command_arguments': '',
'version': '1.0',
'input_data': [{
'name': 'foo',
'type': 'file',
'required': True,
}],
'output_data': [{
'name': 'some_output',
'type': 'file',
}]
}
for output_name in good_names:
definition['output_data'][0]['name'] = output_name
try:
JobInterface(definition)
except InvalidInterfaceDefinition:
self.fail(
'Unable to parse a good interface definition with output name: %s'
% output_name)
for output_name in bad_names:
definition['output_data'][0]['name'] = output_name
try:
JobInterface(definition)
self.fail(
'job interface with a bad output name (%s) was able to get past validation'
% output_name)
except InvalidInterfaceDefinition:
pass
def test_command_param_will_fail_without_input(self):
definition = {
'command': 'test-command',
'command_arguments': '${param-1}',
'version': '1.0',
}
try:
JobInterface(definition)
self.fail('Expected invalid job definition to throw an exception')
except InvalidInterfaceDefinition:
pass
def test_populate_mounts(self):
"""Tests the addition of mount volumes to the configuration."""
exe_config = ExecutionConfiguration()
config_dict = {
'version': '2.0',
'mounts': {
'mount_1': {
'type': 'host',
'host_path': '/host/path'
},
'mount_2': {
'type': 'volume',
'driver': 'x-driver',
'driver_opts': {
'foo': 'bar'
}
}
}
}
interface_dict = {
'version':
'1.4',
'command':
'the cmd',
'command_arguments':
'foo',
'mounts': [{
'name': 'mount_1',
'path': '/mount_1',
'mode': 'ro'
}, {
'name': 'mount_2',
'path': '/mount_2',
'mode': 'rw'
}]
}
job_exe = MagicMock()
job_exe.get_job_configuration.return_value = JobConfiguration(
config_dict)
job_exe.get_job_interface.return_value = JobInterface(interface_dict)
job_exe.get_cluster_id.return_value = 'scale_1234'
exe_config.populate_mounts(job_exe)
docker_params = exe_config.get_job_task_docker_params()
self.assertEqual(docker_params[0].flag, 'volume')
self.assertEqual(docker_params[0].value, '/host/path:/mount_1:ro')
self.assertEqual(docker_params[1].flag, 'volume')
mount_2 = '$(docker volume create --name scale_1234_mount_mount_2 --driver x-driver --opt foo=bar):/mount_2:rw'
self.assertEqual(docker_params[1].value, mount_2)
def test_successful(self):
'''Tests calling JobInterface.validate_connection() successfully.'''
job_interface_dict = {
u'command': u'simple-command',
u'command_arguments': u'',
u'version': u'1.0',
u'input_data': [{u'name': u'Input 1', u'type': u'property'},
{u'name': u'Input 2', u'type': u'file', u'media_types': [u'text/plain']}],
u'output_data': [{u'name': u'Output 1', u'type': u'file'}]
}
job_interface = JobInterface(job_interface_dict)
job_conn = JobConnection()
job_conn.add_property(u'Input 1')
job_conn.add_input_file(u'Input 2', False, [u'text/plain'], False)
job_conn.add_workspace()
# No exception is success
job_interface.validate_connection(job_conn)
def test_complex_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${-f :prop1}'
job_interface_dict['input_data'] = [{
'name': 'prop1',
'type': 'property',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'prop1',
'value': 'property-value',
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = job_environment_dict
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, '-f property-value', 'expected a different command from pre_steps')
def test_property_in_command(self):
job_interface_dict, job_data_dict, job_environment_dict = self._get_simple_interface_data_env()
job_interface_dict['command_arguments'] = '${prop1}'
job_interface_dict['input_data'] = [{
'name': 'prop1',
'type': 'property',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'prop1',
'value': 'property-value',
})
job_interface = JobInterface(job_interface_dict)
job_data = JobData(job_data_dict)
job_environment = JobEnvironment(job_environment_dict)
job_exe_id = 1
job_interface.perform_pre_steps(job_data, job_environment, 1)
job_command_arguments = job_interface.fully_populate_command_argument(job_data, job_environment, job_exe_id)
self.assertEqual(job_command_arguments, 'property-value', 'expected a different command from pre_steps')
def test_successful(self):
"""Tests calling ParseTriggerRuleConfiguration.validate_trigger_for_job() successfully with no warnings"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = ParseTriggerRuleConfiguration(PARSE_TYPE, json.loads(rule_json_str))
interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["text/plain", "application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
job_interface = JobInterface(json.loads(interface_json_str))
warnings = rule_config.validate_trigger_for_job(job_interface)
self.assertListEqual(warnings, [])
def test_media_type_warning(self):
"""Tests calling IngestTriggerRuleConfiguration.validate_trigger_for_job() with a warning for a mis-matched media type"""
rule_json_str = '{"version": "1.0", "condition": {"media_type": "text/plain", "data_types": ["A", "B"]}, "data": {"input_data_name": "my_input", "workspace_name": "my_workspace"}}'
rule_config = IngestTriggerRuleConfiguration(INGEST_TYPE, json.loads(rule_json_str))
interface_json_str = '{"version": "1.0", "command": "my cmd", "command_arguments": "cmd args", "input_data": [{"name": "my_input", "type": "file", "media_types": ["application/json"]}], "output_data": [{"name": "my_output", "type": "file"}]}'
job_interface = JobInterface(json.loads(interface_json_str))
warnings = rule_config.validate_trigger_for_job(job_interface)
self.assertEqual(len(warnings), 1)
def test_parse_data(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict= self._get_simple_interface_data()
job_interface_dict[u'input_data'] = [{"name":"input_file", "type":"file", "required": True}]
job_data_dict[u'input_data'].append({"name":"input_file", "file_id":1})
geo_json = {"type": 'Feature'}
geo_metadata = {
"data_started": '2015-01-01T00:00:00Z',
"geo_json": geo_json
}
results_manifest={"version" : "1.1", "parse_results" : [{"filename":"/some/path/foo.txt", "geo_metadata":geo_metadata}]}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = u''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.save_parse_results.assert_called_with({u'/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None)})
def test_parse_data(self, mock_loads, mock_open, mock_exists):
job_interface_dict, job_data_dict = self._get_simple_interface_data()
job_interface_dict['input_data'] = [{
'name': 'input_file',
'type': 'file',
'required': True,
}]
job_data_dict['input_data'].append({
'name': 'input_file',
'file_id': self.file.id,
})
geo_json = {'type': 'Feature'}
geo_metadata = {
'data_started': '2015-01-01T00:00:00Z',
'geo_json': geo_json
}
results_manifest = {
'version': '1.1',
'parse_results': [{
'filename': '/some/path/foo.txt',
'geo_metadata': geo_metadata,
}]
}
mock_loads.return_value = results_manifest
mock_exists.return_value = True
job_exe = MagicMock()
job_interface = JobInterface(job_interface_dict)
job_data = Mock(spec=JobData)
job_data.save_parse_results = Mock()
fake_stdout = ''
job_interface.perform_post_steps(job_exe, job_data, fake_stdout)
job_data.save_parse_results.assert_called_with({
'/some/path/foo.txt': (geo_json, '2015-01-01T00:00:00Z', None, [], None, None),
})
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
class TestJobTypeManagerEditJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface)
new_valid_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['application/json'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.new_valid_job_interface = JobInterface(new_valid_interface)
new_invalid_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]}
self.new_invalid_job_interface = JobInterface(new_invalid_interface)
self.definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type.name,
'version': self.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}]
}
self.recipe_def = RecipeDefinition(self.definition)
self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def,
None)
def test_valid_interface(self):
"""Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and a valid interface change
is made"""
# Call test
JobType.objects.edit_job_type(self.job_type.id, self.new_valid_job_interface)
# Check results
job_type = JobType.objects.get(pk=self.job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_valid_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
# New revision due to interface change
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_invalid_interface(self):
"""Tests calling JobTypeManager.edit_job_type() where the job type is in a recipe and an invalid interface
change is made"""
# Call test
self.assertRaises(InvalidDefinition, JobType.objects.edit_job_type, self.job_type.id,
self.new_invalid_job_interface)
# Check results
job_type = JobType.objects.get(pk=self.job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.job_type = JobType.objects.create_job_type('name', '1.0', self.job_interface)
new_valid_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['application/json'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.new_valid_job_interface = JobInterface(new_valid_interface)
new_invalid_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]}
self.new_invalid_job_interface = JobInterface(new_invalid_interface)
self.definition = {
'version': '1.0',
'input_data': [{
'name': 'Recipe Input',
'type': 'file',
'media_types': ['text/plain'],
}],
'jobs': [{
'name': 'Job 1',
'job_type': {
'name': self.job_type.name,
'version': self.job_type.version,
},
'recipe_inputs': [{
'recipe_input': 'Recipe Input',
'job_input': 'Test Input 1',
}]
}]
}
self.recipe_def = RecipeDefinition(self.definition)
self.recipe = RecipeType.objects.create_recipe_type('name', '1.0', 'title', 'description', self.recipe_def,
None)
class TestJobTypeManagerEditJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
new_interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 2',
'type': 'files',
'media_types': ['image/png', 'image/tiff'],
}],
'output_data': [{
'name': 'Test Output 2',
'type': 'file',
}]}
self.new_job_interface = JobInterface(new_interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.new_configuration = {
'version': '1.0',
'condition': {
'media_type': 'application/json'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.new_trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE,
self.new_configuration)
def test_change_general_fields(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to some general fields'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
priority = 12
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
new_title = 'my new title'
new_priority = 13
new_error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-16': self.error.name,
}
})
new_is_paused = True
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title,
priority=priority, error_mapping=error_mapping)
# Call test
JobType.objects.edit_job_type(job_type.id, title=new_title, priority=new_priority,
error_mapping=new_error_mapping, is_paused=new_is_paused)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
self.assertEqual(job_type.title, new_title)
self.assertEqual(job_type.priority, new_priority)
self.assertDictEqual(job_type.get_error_interface().get_dict(), new_error_mapping.get_dict())
self.assertEqual(job_type.is_paused, new_is_paused)
self.assertIsNotNone(job_type.paused)
def test_change_to_interface(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to the interface'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, self.new_job_interface, None, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
# New revision due to interface change
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_change_to_trigger_rule(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to the trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.new_trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, None, new_trigger_rule, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertFalse(trigger_rule.is_active)
new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id)
self.assertTrue(new_trigger_rule.is_active)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def test_remove_trigger_rule(self):
'''Tests calling JobTypeManager.edit_job_type() that removes the trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, None, None, True)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertFalse(trigger_rule.is_active)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def test_change_to_both(self):
'''Tests calling JobTypeManager.edit_job_type() with a change to both the definition and the trigger rule
'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.new_trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
JobType.objects.edit_job_type(job_type.id, self.new_job_interface, new_trigger_rule, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.new_job_interface.get_dict())
self.assertEqual(job_type.revision_num, 2)
self.assertEqual(job_type.trigger_rule_id, new_trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertFalse(trigger_rule.is_active)
new_trigger_rule = TriggerRule.objects.get(pk=new_trigger_rule.id)
self.assertTrue(new_trigger_rule.is_active)
# New revision due to definition change
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 2)
def test_invalid_trigger_rule(self):
'''Tests calling JobTypeManager.edit_job_type() with a new invalid trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
new_trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
configuration=self.new_trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule)
# Call test
self.assertRaises(InvalidConnection, JobType.objects.edit_job_type, job_type.id, self.new_job_interface,
new_trigger_rule, False)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
num_of_revs = JobTypeRevision.objects.filter(job_type_id=job_type.id).count()
self.assertEqual(num_of_revs, 1)
def test_system_job_type(self):
'''Tests calling JobTypeManager.edit_job_type() for a system job type'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
new_title = 'my new title'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title)
job_type.is_system = True
job_type.save()
# Call test
self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
# No change
self.assertEqual(job_type.title, title)
def test_uneditable_field(self):
'''Tests calling JobTypeManager.edit_job_type() to change an uneditable field'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
new_title = 'my new title'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, title=title)
# Call test
self.assertRaises(Exception, JobType.objects.edit_job_type, job_type.id, title=new_title, is_system=True)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
# No change
self.assertEqual(job_type.title, title)
def test_invalid_error_mapping(self):
'''Tests calling JobTypeManager.edit_job_type() with an invalid error mapping'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'1': 'test-invalid-error',
}
})
# Call test
self.assertRaises(Exception, JobType.objects.edit_job_type, name, version, self.job_interface,
error_mapping=error_mapping, title=title, description=description, priority=priority,
is_system=is_system)
class TestJobTypeManagerCreateJobType(TransactionTestCase):
def setUp(self):
django.setup()
self.workspace = storage_test_utils.create_workspace()
self.error = error_test_utils.create_error()
interface = {
'version': '1.0',
'command': 'my_command',
'command_arguments': 'args',
'input_data': [{
'name': 'Test Input 1',
'type': 'file',
'media_types': ['text/plain'],
}],
'output_data': [{
'name': 'Test Output 1',
'type': 'files',
'media_type': 'image/png',
}]}
self.job_interface = JobInterface(interface)
self.configuration = {
'version': '1.0',
'condition': {
'media_type': 'text/plain'
},
'data': {
'input_data_name': 'Test Input 1',
'workspace_name': self.workspace.name
}
}
self.trigger_config = job_test_utils.MockTriggerRuleConfiguration(job_test_utils.MOCK_TYPE, self.configuration)
self.error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'-15': self.error.name,
}
})
def test_successful_no_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() successfully with no trigger rule or error mapping'''
name = 'my-job-type'
version = '1.0'
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
def test_successful_with_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() successfully with a trigger rule and error mapping'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_TYPE,
configuration=self.trigger_config.get_dict())
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, trigger_rule, self.error_mapping)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertEqual(job_type.trigger_rule_id, trigger_rule.id)
trigger_rule = TriggerRule.objects.get(pk=trigger_rule.id)
self.assertTrue(trigger_rule.is_active)
self.assertDictEqual(job_type.get_error_interface().get_dict(), self.error_mapping.get_dict())
def test_invalid_trigger_rule(self):
'''Tests calling JobTypeManager.create_job_type() with an invalid trigger rule'''
name = 'my-job-type'
version = '1.0'
trigger_rule = trigger_test_utils.create_trigger_rule(trigger_type=job_test_utils.MOCK_ERROR_TYPE,
configuration=self.trigger_config.get_dict())
# Call test
self.assertRaises(InvalidConnection, JobType.objects.create_job_type, name, version, self.job_interface,
trigger_rule, self.error_mapping)
def test_successful_other_fields(self):
'''Tests calling JobTypeManager.create_job_type() successfully with additional fields'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
description=description, priority=priority)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
self.assertEqual(job_type.description, description)
self.assertEqual(job_type.priority, priority)
self.assertIsNone(job_type.archived)
self.assertIsNone(job_type.paused)
def test_successful_paused(self):
'''Tests calling JobTypeManager.create_job_type() and pausing it'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_paused = True
# Call test
job_type = JobType.objects.create_job_type(name, version, self.job_interface, title=title,
description=description, priority=priority, is_paused=is_paused)
# Check results
job_type = JobType.objects.select_related('trigger_rule').get(pk=job_type.id)
self.assertDictEqual(job_type.get_job_interface().get_dict(), self.job_interface.get_dict())
self.assertEqual(job_type.revision_num, 1)
self.assertIsNone(job_type.trigger_rule_id)
self.assertDictEqual(job_type.get_error_interface().get_dict(), ErrorInterface(None).get_dict())
self.assertEqual(job_type.description, description)
self.assertEqual(job_type.priority, priority)
self.assertEqual(job_type.is_paused, is_paused)
self.assertIsNotNone(job_type.paused)
def test_uneditable_field(self):
'''Tests calling JobTypeManager.create_job_type() with an uneditable field'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
# Call test
self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface, title=title,
description=description, priority=priority, is_system=is_system)
def test_invalid_error_mapping(self):
'''Tests calling JobTypeManager.create_job_type() with an invalid error mapping'''
name = 'my-job-type'
version = '1.0'
title = 'my title'
description = 'my-description'
priority = 13
is_system = True
error_mapping = ErrorInterface({
'version': '1.0',
'exit_codes': {
'1': 'test-invalid-error',
}
})
# Call test
self.assertRaises(Exception, JobType.objects.create_job_type, name, version, self.job_interface,
error_mapping=error_mapping, title=title, description=description, priority=priority,
is_system=is_system)
