def test_validate_raises_general_bespin_exception(self):
mock_api = Mock()
mock_api.job_template_validate.side_effect = BespinException('Something failed')
job_template = JobTemplate(tag='sometag/v1/human', name='myjob', fund_code='001', job_order={})
with self.assertRaises(BespinException) as raised_exception:
job_template.validate(mock_api)
self.assertEqual(str(raised_exception.exception), 'Something failed')
def test_validate_flattens_bespin_dict_exception(self):
mock_api = Mock()
mock_api.job_template_validate.side_effect = BespinClientErrorException(json.dumps({
"field1": ["reason1", "reason2"],
"field2": ["reason3"]
}))
job_template = JobTemplate(tag='sometag/v1/human', name='myjob', fund_code='001', job_order={})
with self.assertRaises(IncompleteJobTemplateException) as raised_exception:
job_template.validate(mock_api)
self.assertEqual(str(raised_exception.exception), 'field1: reason1\nfield1: reason2\nfield2: reason3')
def test_get_formatted_dict(self):
mock_api = Mock()
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={})
expected_dict = {
'fund_code': '001',
'job_order': {},
'name': 'myjob',
'tag': 'sometag'
}
self.assertEqual(job_template.get_formatted_dict(api=mock_api), expected_dict)
mock_api.job_strategy_get_for_name.assert_not_called()
def test_get_dds_files_details(self, mock_dds_file_util):
mock_dds_file_util.return_value.find_file_for_path.return_value = 'filedata1'
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={
'myfile': {
'class': 'File',
'path': 'dds://project_somepath.txt'
},
'myint': 123
})
file_details = job_template.get_dds_files_details()
self.assertEqual(file_details, [('filedata1', 'dds_project_somepath.txt')])
def test_get_formatted_dict__job_strategy(self):
mock_api = Mock()
mock_api.job_strategy_get_for_name.return_value = {'id':'abc123'}
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={}, job_strategy='cloud3')
expected_dict = {
'fund_code': '001',
'job_order': {},
'name': 'myjob',
'tag': 'sometag',
'job_strategy': 'abc123',
}
self.assertEqual(job_template.get_formatted_dict(api=mock_api), expected_dict)
mock_api.job_strategy_get_for_name.assert_called_with('cloud3')
def test_create_user_job_order_does_not_modify_original(self):
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={
'myfile': {
'class': 'File',
'path': 'dds://project/somepath.txt'
}
})
job_template.create_user_job_order()
self.assertEqual(job_template.job_order, {
'myfile': {
'class': 'File',
'path': 'dds://project/somepath.txt',
}
})
def test_create_job(self, mock_dds_file_util):
mock_dds_file_util.return_value.find_file_for_path.return_value = 'filedata1'
mock_api = Mock()
mock_api.dds_user_credentials_list.return_value = [{'id': 111, 'dds_id': 112}]
mock_api.workflow_configurations_list.return_value = [
{
'id': 222
}
]
mock_api.stage_group_post.return_value = {
'id': 333
}
job_template = JobTemplate(tag='sometag/v1/human', name='myjob', fund_code='001', job_order={
'myfile': {
'class': 'File',
'path': 'dds://project_somepath.txt'
},
'myint': 555
})
job_template.get_dds_files_details = Mock()
mock_file = Mock(project_id=666, current_version={'upload': {'size': 4002}})
mock_file.id = 777
job_template.get_dds_files_details.return_value = [[mock_file, 'somepath']]
job_template.create_job(mock_api)
mock_api.workflow_configurations_list.assert_called_with(tag='human', workflow_tag='sometag')
mock_api.dds_job_input_files_post.assert_called_with(666, 777, 'somepath', 0, 0, 111, stage_group_id=333,
size=4002)
mock_api.job_templates_create_job.assert_called_with({
'name': 'myjob',
'fund_code': '001',
'job_order': {
'myfile': {
'class': 'File',
'path': 'dds_project_somepath.txt'
},
'myint': 555
},
'tag': 'sometag/v1/human',
'stage_group': 333
})
mock_dds_file_util.return_value.give_download_permissions.assert_called_with(666, 112)
def test_validate(self, mock_job_order_format_files, mock_dds_file_util):
mock_dds_file_util.return_value.find_file_for_path.return_value = 'filedata1'
mock_api = Mock()
mock_api.dds_user_credentials_list.return_value = [{'id': 111, 'dds_id': 112}]
mock_api.workflow_configurations_list.return_value = [
{
'id': 222
}
]
mock_api.stage_group_post.return_value = {
'id': 333
}
job_order = {
'myfile': {
'class': 'File',
'path': 'dds://project_somepath.txt'
},
'myint': 555
}
job_template = JobTemplate(tag='sometag/v1/human', name='myjob', fund_code='001', job_order=job_order)
job_template.get_dds_files_details = Mock()
mock_file = Mock(project_id=666, current_version={'upload': {'size': 4002}})
mock_file.id = 777
job_template.get_dds_files_details.return_value = [[mock_file, 'somepath']]
job_template.validate(mock_api)
mock_api.job_template_validate.assert_called_with({
'name': 'myjob',
'fund_code': '001',
'job_order': {
'myfile': {
'class': 'File',
'path': 'dds://project_somepath.txt'
},
'myint': 555},
'tag': 'sometag/v1/human'
})
job_template.get_dds_files_details.assert_called_with()
mock_job_order_format_files.return_value.walk.assert_called_with(job_order)
def test_create_user_job_order_json(self):
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={
'myfile': {
'class': 'File',
'path': 'dds://project/somepath.txt'
},
'my_path_file': {
'class': 'File',
'path': '/tmp/data.txt'
},
'my_url_file': {
'class': 'File',
'location': 'https://github.com/datafile1.dat'
},
'myint': 123,
'myfileary': [
{
'class': 'File',
'path': 'dds://project/somepath1.txt'
},
{
'class': 'File',
'path': 'dds://project/somepath2.txt'
},
],
'myfastq_pairs': [
{'file1':
{'class': 'File',
'path': 'dds://myproject/rawData/SAAAA_R1_001.fastq.gz'
},
'file2': {
'class': 'File',
'path': 'dds://myproject/rawData/SAAAA_R2_001.fastq.gz'
},
'name': 'Sample1'}]
})
user_job_order = job_template.create_user_job_order()
self.assertEqual(user_job_order['myint'], 123)
self.assertEqual(user_job_order['myfile'], {
'class': 'File',
'path': 'dds_project_somepath.txt'
})
self.assertEqual(user_job_order['myfileary'], [
{
'class': 'File',
'path': 'dds_project_somepath1.txt'
},
{
'class': 'File',
'path': 'dds_project_somepath2.txt'
},
])
self.assertEqual(user_job_order['myfastq_pairs'], [
{'file1':
{'class': 'File',
'path': 'dds_myproject_rawData_SAAAA_R1_001.fastq.gz'
},
'file2': {
'class': 'File',
'path': 'dds_myproject_rawData_SAAAA_R2_001.fastq.gz'
},
'name': 'Sample1'}])
self.assertEqual(user_job_order['my_path_file'], {
'class': 'File',
'path': '/tmp/data.txt'
}, "Plain file paths should not be modified.")
self.assertEqual(user_job_order['my_url_file'], {
'class': 'File',
'location': 'https://github.com/datafile1.dat'
}, "URL file paths should not be modified.")
def test_constructor__job_strategy(self):
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={})
self.assertEqual(job_template.job_strategy, None)
job_template = JobTemplate(tag='sometag', name='myjob', fund_code='001', job_order={}, job_strategy='cloud2')
self.assertEqual(job_template.job_strategy, 'cloud2')