def test_can_stage_delivery(self):
url = "/".join([
self.API_BASE, "stage", "runfolder",
"160930_ST-E00216_0111_BH37CWALXX"
])
response = self.fetch(url, method='POST', body='')
self.assertEqual(response.code, 202)
response_json = json.loads(response.body)
staging_status_links = response_json.get("staging_order_links")
for link in staging_status_links:
def _get_delivery_status():
self.http_client.fetch(link, self.stop)
status_response = self.wait()
return json.loads(status_response.body)["status"]
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=_get_delivery_status,
expected=StagingStatus.staging_successful.name)
def test_can_stage_and_deliver_batch_flowcells(self):
with tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
prefix='160930_ST-E00216_0555_BH37CWALXX_') as tmpdir1, \
tempfile.TemporaryDirectory(dir='./tests/resources/runfolders/',
prefix='160930_ST-E00216_0556_BH37CWALXX_') as tmpdir2:
self._create_projects_dir_with_random_data(tmpdir1, 'XYZ_123')
self._create_projects_dir_with_random_data(tmpdir2, 'XYZ_123')
url = "/".join(
[self.API_BASE, "stage", "project", 'runfolders', 'XYZ_123'])
payload = {'delivery_mode': 'BATCH'}
response = self.fetch(url, method='POST', body=json.dumps(payload))
self.assertEqual(response.code, 202)
payload = {'delivery_mode': 'BATCH'}
response_failed = self.fetch(url,
method='POST',
body=json.dumps(payload))
self.assertEqual(response_failed.code, 403)
response_json = json.loads(response.body)
staging_status_links = response_json.get("staging_order_links")
for project, link in staging_status_links.items():
self.assertEqual(project, 'XYZ_123')
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, link),
expected=StagingStatus.staging_successful.name)
def test_deliver_by_staging_id(self):
source = '/foo/bar'
staging_target = '/staging/dir/bar'
staging_order = StagingOrder(source=source, staging_target=staging_target)
staging_order.status = StagingStatus.staging_successful
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
self.mock_staging_service.get_delivery_order_by_id.return_value = self.delivery_order
with patch('shutil.rmtree') as mock_rmtree:
res = yield self.dds_service.deliver_by_staging_id(
staging_id=1,
delivery_project='snpseq00001',
token_path='token_path',
md5sum_file='md5sum_file')
mock_rmtree.assert_called_once_with(staging_target)
def _get_delivery_order():
return self.delivery_order.delivery_status
assert_eventually_equals(self, 1, _get_delivery_order, DeliveryStatus.delivery_successful)
self.mock_mover_runner.run.assert_called_with([
'dds',
'--token-path', 'token_path',
'--log-file', '/foo/bar/log',
'--no-prompt',
'data', 'put',
'--mount-dir', '/foo/bar/staging_dir',
'--source', '/staging/dir/bar',
'--project', 'snpseq00001',
'--silent'
])
def test_stage_order(self):
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status,
StagingStatus.staging_successful)
def test_exception_in_staging_order(self):
def raise_exception(x):
raise Exception
self.mock_external_runner_service.wait_for_execution = raise_exception
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status, StagingStatus.staging_failed)
def test_stage_order(self):
res = yield self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
def _get_staging_size():
return self.staging_order1.size
assert_eventually_equals(self, 1, _get_stating_status, StagingStatus.staging_successful)
self.assertEqual(self.staging_order1.size, 207707566)
def test_exception_in_staging_order(self):
mock_external_runner_service = self.MockExternalRunnerService(
throw=True)
self.staging_service.external_program_service = mock_external_runner_service
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status,
StagingStatus.staging_failed)
def test_unsuccessful_staging_order(self):
mock_external_runner_service = self.MockExternalRunnerService(
return_status=1)
self.staging_service.external_program_service = mock_external_runner_service
self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status,
StagingStatus.staging_failed)
def test_unsuccessful_staging_order(self):
@coroutine
def wait_as_coroutine(x):
return ExecutionResult(stdout="", stderr="", status_code=1)
self.mock_external_runner_service.wait_for_execution = wait_as_coroutine
yield self.staging_service.stage_order(stage_order=self.staging_order1)
def _get_stating_status():
return self.staging_order1.status
assert_eventually_equals(self, 1, _get_stating_status, StagingStatus.staging_failed)
def test_can_stage_and_delivery_project_dir(self):
# Note that this is a test which skips mover (since to_outbox is not expected to be installed on the system
# where this runs)
with tempfile.TemporaryDirectory(
dir='./tests/resources/projects') as tmp_dir:
dir_name = os.path.basename(tmp_dir)
url = "/".join([self.API_BASE, "stage", "project", dir_name])
response = self.fetch(url, method='POST', body='')
self.assertEqual(response.code, 202)
response_json = json.loads(response.body)
staging_status_links = response_json.get("staging_order_links")
for project, link in staging_status_links.items():
self.assertEqual(project, dir_name)
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, link),
expected=StagingStatus.staging_successful.name)
staging_order_project_and_id = response_json.get(
"staging_order_ids")
for project, staging_id in staging_order_project_and_id.items():
delivery_url = '/'.join(
[self.API_BASE, 'deliver', 'stage_id',
str(staging_id)])
delivery_body = {
'delivery_project_id': 'fakedeliveryid2016',
'skip_mover': True
}
delivery_resp = self.fetch(delivery_url,
method='POST',
body=json.dumps(delivery_body))
delivery_resp_as_json = json.loads(delivery_resp.body)
delivery_link = delivery_resp_as_json['delivery_order_link']
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, delivery_link),
expected=DeliveryStatus.delivery_skipped.name)
def test_deliver_by_staging_id(self):
staging_order = StagingOrder(source='/foo/bar', staging_target='/staging/dir/bar')
staging_order.status = StagingStatus.staging_successful
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
self.mock_staging_service.get_delivery_order_by_id.return_value = self.delivery_order
res = yield self.mover_delivery_service.deliver_by_staging_id(staging_id=1,
delivery_project='xyz123',
md5sum_file='md5sum_file')
def _get_delivery_order():
return self.delivery_order.delivery_status
assert_eventually_equals(self, 1, _get_delivery_order, DeliveryStatus.delivery_in_progress)
self.mock_mover_runner.run.assert_called_once_with(['/foo/bar/to_outbox', '/foo', 'TestProj'])
def test_possible_to_delivery_by_staging_id_and_skip_mover(self):
staging_order = StagingOrder(source='/foo/bar', staging_target='/staging/dir/bar')
staging_order.status = StagingStatus.staging_successful
self.mock_staging_service.get_stage_order_by_id.return_value = staging_order
self.mock_staging_service.get_delivery_order_by_id.return_value = self.delivery_order
self.mover_delivery_service.deliver_by_staging_id(staging_id=1,
delivery_project='xyz123',
md5sum_file='md5sum_file',
skip_mover=True)
def _get_delivery_order():
return self.delivery_order.delivery_status
assert_eventually_equals(self, 1, _get_delivery_order, DeliveryStatus.delivery_skipped)
def test_can_stage_and_delivery_runfolder(self):
# Note that this is a test which skips mover (since to_outbox is not expected to be installed on the system
# where this runs)
url = "/".join([
self.API_BASE, "stage", "runfolder",
"160930_ST-E00216_0111_BH37CWALXX"
])
response = self.fetch(url, method='POST', body='')
self.assertEqual(response.code, 202)
response_json = json.loads(response.body)
staging_status_links = response_json.get("staging_order_links")
for project, link in staging_status_links.items():
self.assertEqual(project, "ABC_123")
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, link),
expected=StagingStatus.staging_successful.name)
# The size of the fake project is 15 bytes
assert_eventually_equals(self,
timeout=5,
delay=1,
f=partial(self._get_size, link),
expected=15)
staging_order_project_and_id = response_json.get("staging_order_ids")
for project, staging_id in staging_order_project_and_id.items():
delivery_url = '/'.join(
[self.API_BASE, 'deliver', 'stage_id',
str(staging_id)])
delivery_body = {
'delivery_project_id': 'fakedeliveryid2016',
'skip_mover': True
}
delivery_resp = self.fetch(delivery_url,
method='POST',
body=json.dumps(delivery_body))
delivery_resp_as_json = json.loads(delivery_resp.body)
delivery_link = delivery_resp_as_json['delivery_order_link']
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, delivery_link),
expected=DeliveryStatus.delivery_skipped.name)
def test_can_stage_and_delivery_runfolder(self):
# Note that this is a test which skips mover (since to_outbox is not expected to be installed on the system
# where this runs)
with tempfile.TemporaryDirectory(
dir='./tests/resources/runfolders/',
prefix='160930_ST-E00216_0111_BH37CWALXX_') as tmp_dir:
dir_name = os.path.basename(tmp_dir)
self._create_projects_dir_with_random_data(tmp_dir)
self._create_checksums_file(tmp_dir)
url = "/".join([self.API_BASE, "stage", "runfolder", dir_name])
response = self.fetch(url, method='POST', body='')
self.assertEqual(response.code, 202)
response_json = json.loads(response.body)
staging_status_links = response_json.get("staging_order_links")
for project, link in staging_status_links.items():
self.assertEqual(project, "ABC_123")
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, link),
expected=StagingStatus.staging_successful.name)
# The size of the fake project is 1024 bytes
assert_eventually_equals(self,
timeout=5,
delay=1,
f=partial(self._get_size, link),
expected=1024)
staging_order_project_and_id = response_json.get(
"staging_order_ids")
for project, staging_id in staging_order_project_and_id.items():
delivery_url = '/'.join(
[self.API_BASE, 'deliver', 'stage_id',
str(staging_id)])
delivery_body = {
'delivery_project_id': 'fakedeliveryid2016',
'skip_mover': True
}
delivery_resp = self.fetch(delivery_url,
method='POST',
body=json.dumps(delivery_body))
delivery_resp_as_json = json.loads(delivery_resp.body)
delivery_link = delivery_resp_as_json['delivery_order_link']
assert_eventually_equals(
self,
timeout=5,
delay=1,
f=partial(self._get_delivery_status, delivery_link),
expected=DeliveryStatus.delivery_skipped.name)
