def test_done(self, reader):
api = mock.MagicMock()
runner = mock.MagicMock()
runner.api = api
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
reader().open.return_value = []
api.collections().list().execute.side_effect = ({
"items": []
}, {
"items": [{
"manifest_text": "XYZ"
}]
})
arvjob = arvados_cwl.ArvadosJob(runner)
arvjob.name = "testjob"
arvjob.builder = mock.MagicMock()
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.done({
"state": "Complete",
"output": "99999999999999999999999999999993+99",
"log": "99999999999999999999999999999994+99",
"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
})
api.collections().list.assert_has_calls([
mock.call(),
mock.call(
filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
[
'portable_data_hash', '=',
'99999999999999999999999999999993+99'
], ['name', '=', 'Output 9999999 of testjob']]),
mock.call().execute(num_retries=0),
mock.call(limit=1,
filters=[[
'portable_data_hash', '=',
'99999999999999999999999999999993+99'
]],
select=['manifest_text']),
mock.call().execute(num_retries=0)
])
api.collections().create.assert_called_with(
ensure_unique_name=True,
body={
'portable_data_hash': '99999999999999999999999999999993+99',
'manifest_text': 'XYZ',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'name': 'Output 9999999 of testjob'
})
def test_done_use_existing_collection(self, reader):
api = mock.MagicMock()
runner = mock.MagicMock()
runner.api = api
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
reader().open.return_value = StringIO.StringIO(
"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
""")
api.collections().list().execute.side_effect = (
{"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
{"items": [{"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"}]},
)
arvjob = arvados_cwl.ArvadosJob(runner,
mock.MagicMock(),
{},
None,
[],
[],
"testjob")
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.collect_outputs.return_value = {"out": "stuff"}
arvjob.done({
"state": "Complete",
"output": "99999999999999999999999999999993+99",
"log": "99999999999999999999999999999994+99",
"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
})
api.collections().list.assert_has_calls([
mock.call(),
# Output collection
mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
['portable_data_hash', '=', '99999999999999999999999999999993+99'],
['name', '=', 'Output 9999999 of testjob']]),
mock.call().execute(num_retries=0),
# Log collection
mock.call(filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
['portable_data_hash', '=', '99999999999999999999999999999994+99'],
['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
mock.call().execute(num_retries=0)
])
self.assertFalse(api.collections().create.called)
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
def test_done_use_existing_collection(self, reader):
api = mock.MagicMock()
runner = mock.MagicMock()
runner.api = api
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
reader().open.return_value = []
api.collections().list().execute.side_effect = ({
"items": [{
"uuid": "zzzzz-4zz18-zzzzzzzzzzzzzz2"
}]
}, )
arvjob = arvados_cwl.ArvadosJob(runner)
arvjob.name = "testjob"
arvjob.builder = mock.MagicMock()
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.done({
"state": "Complete",
"output": "99999999999999999999999999999993+99",
"log": "99999999999999999999999999999994+99",
"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
})
api.collections().list.assert_has_calls([
mock.call(),
mock.call(
filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
[
'portable_data_hash', '=',
'99999999999999999999999999999993+99'
], ['name', '=', 'Output 9999999 of testjob']]),
mock.call().execute(num_retries=0)
])
self.assertFalse(api.collections().create.called)
def test_done(self, reader):
api = mock.MagicMock()
runner = mock.MagicMock()
runner.api = api
runner.project_uuid = "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
runner.num_retries = 0
runner.ignore_docker_for_reuse = False
reader().open.return_value = StringIO.StringIO(
"""2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.tmpdir)=/tmp/crunch-job-task-work/compute3.1/tmpdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.outdir)=/tmp/crunch-job-task-work/compute3.1/outdir
2016-11-02_23:12:18 c97qk-8i9sb-cryqw2blvzy4yaj 13358 0 stderr 2016/11/02 23:12:18 crunchrunner: $(task.keep)=/keep
""")
api.collections().list().execute.side_effect = ({
"items": []
}, {
"items": [{
"manifest_text": "XYZ"
}]
}, {
"items": []
}, {
"items": [{
"manifest_text": "ABC"
}]
})
arvjob = arvados_cwl.ArvadosJob(runner)
arvjob.name = "testjob"
arvjob.builder = mock.MagicMock()
arvjob.output_callback = mock.MagicMock()
arvjob.collect_outputs = mock.MagicMock()
arvjob.collect_outputs.return_value = {"out": "stuff"}
arvjob.done({
"state": "Complete",
"output": "99999999999999999999999999999993+99",
"log": "99999999999999999999999999999994+99",
"uuid": "zzzzz-8i9sb-zzzzzzzzzzzzzzz"
})
api.collections().list.assert_has_calls([
mock.call(),
# Output collection check
mock.call(
filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
[
'portable_data_hash', '=',
'99999999999999999999999999999993+99'
], ['name', '=', 'Output 9999999 of testjob']]),
mock.call().execute(num_retries=0),
mock.call(limit=1,
filters=[[
'portable_data_hash', '=',
'99999999999999999999999999999993+99'
]],
select=['manifest_text']),
mock.call().execute(num_retries=0),
# Log collection's turn
mock.call(
filters=[['owner_uuid', '=', 'zzzzz-8i9sb-zzzzzzzzzzzzzzz'],
[
'portable_data_hash', '=',
'99999999999999999999999999999994+99'
],
['name', '=', 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz']]),
mock.call().execute(num_retries=0),
mock.call(limit=1,
filters=[[
'portable_data_hash', '=',
'99999999999999999999999999999994+99'
]],
select=['manifest_text']),
mock.call().execute(num_retries=0)
])
api.collections().create.assert_has_calls([
mock.call(ensure_unique_name=True,
body={
'portable_data_hash':
'99999999999999999999999999999993+99',
'manifest_text': 'XYZ',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'name': 'Output 9999999 of testjob'
}),
mock.call().execute(num_retries=0),
mock.call(ensure_unique_name=True,
body={
'portable_data_hash':
'99999999999999999999999999999994+99',
'manifest_text': 'ABC',
'owner_uuid': 'zzzzz-8i9sb-zzzzzzzzzzzzzzz',
'name': 'Log of zzzzz-8i9sb-zzzzzzzzzzzzzzz'
}),
mock.call().execute(num_retries=0),
])
arvjob.output_callback.assert_called_with({"out": "stuff"}, "success")
