def _put_test_bdl(dirpath=os.path.join(TEST_DIR, 'res', 'bundle'),
replica='aws',
staging_bucket='org-humancellatlas-dss-cli-test'):
"""
Implements a context manager that uploads a bundle to the data
store using `hca dss upload` then deletes it when done, if the
user has the requisite permissions.
:param str dirpath: path of the directory to upload (`--src-dir`)
:param str replica: replica to use (`--replica`)
:param str staging_bucket`: passed to `--staging-bucket`
:rtype: dict
:returns: upload response object
"""
put_args = [
'dss', 'upload', '--src-dir', dirpath, '--replica', replica,
'--staging-bucket', staging_bucket, '--no-progress'
]
with CapturingIO('stdout') as stdout:
hca.cli.main(args=put_args)
rv = json.loads(stdout.captured())
yield rv
del_args = [
'dss', 'delete-bundle', '--uuid', rv['bundle_uuid'], '--replica',
replica, '--reason', 'tear down test bundle'
]
try:
with CapturingIO('stdout'):
hca.cli.main(args=del_args)
except hca.util.exceptions.SwaggerAPIException as e:
# Deleting bundles is a privilege, not a right
assert e.code == 403
def test_json_input(self):
"""Ensure that adding JSON input works"""
args = ["dss", "post-search", "--es-query", '{}', "--replica", "aws"]
with CapturingIO('stdout') as stdout:
hca.cli.main(args)
self.assertEqual(json.loads(stdout.captured())["es_query"], {})
def test_list_area_command_with_long_option(self):
self.upload_bucket.Object('/'.join([self.area.uuid,
'file1.fastq.gz'])).put(Body="foo")
self.simulate_credentials_api(area_uuid=self.area.uuid)
list_url = 'https://upload.{stage}.data.humancellatlas.org/v1/area/{uuid}/files_info'.format(
stage=self.deployment_stage, uuid=self.area.uuid)
responses.add(responses.PUT,
list_url,
status=200,
json=[{
"name": "file1.fastq.gz",
"content_type": "binary/octet-stream; dcp-type=data",
"size": 123,
"url": "http://example.com",
"checksums": {
"sha1": "shaaa"
}
}])
with CapturingIO('stdout') as stdout:
ListAreaCommand(Namespace(long=True))
six.assertRegex(self, stdout.captured(), "size\s+123")
six.assertRegex(self, stdout.captured(),
"Content-Type\s+binary/octet-stream; dcp-type=data")
six.assertRegex(self, stdout.captured(), "SHA1\s+shaaa")
def _test_with_mock(self, expected_length, no_pagination=False):
paged_args = ['dss', 'get-bundles-all', '--replica', 'aws']
if no_pagination is True:
paged_args.append('--no-paginate')
with CapturingIO('stdout') as stdout:
hca.cli.main(args=paged_args)
output = json.loads(stdout.captured())
self.assertEqual(expected_length, len(output['results']))
def test_use_selected_area_and_env_if_none_given(self, mock_check_file_statuses):
with CapturingIO('stdout') as stdout:
args = Namespace(env=None, uuid=None, output_file_name=None)
GenerateStatusReportCommand(args)
mock_check_file_statuses.assert_called_once_with(self.area.uuid, self.area.uuid)
assert stdout.captured() == 'File status report for {}/{} generated, located {}.txt\n'.format(
'test', self.area.uuid, self.area.uuid)
def test_correctly_handles_missing_file(self, mock_get_checksum_status):
mock_return_value = 'CHECKSUM_STATUS_RETRIEVAL_ERROR: GET https://upload...website/checksum returned 404'
mock_get_checksum_status.return_value = mock_return_value
with CapturingIO('stdout') as stdout:
args = Namespace(env='dev', uuid='1234', filename='missing_file')
ListFileStatusCommand(args)
assert stdout.captured() == mock_return_value + "\n"
def test_post_search_cli(self):
query = json.dumps({})
replica = "aws"
args = ["dss", "post-search", "--es-query", query, "--replica", replica, "--output-format", "raw"]
with CapturingIO('stdout') as stdout:
hca.cli.main(args)
result = json.loads(stdout.captured())
self.assertIn("results", result)
def test_when_given_an_unrecognized_urn_it_stores_it_in_upload_area_list_and_sets_it_as_current_area(self):
with CapturingIO('stdout') as stdout:
args = Namespace(uri_or_alias=self.uri)
SelectCommand(args)
config = hca.get_config()
self.assertIn(self.area_uuid, config.upload.areas)
self.assertEqual(self.uri, config.upload.areas[self.area_uuid]['uri'])
self.assertEqual(self.area_uuid, config.upload.current_area)
def test_when_given_an_alias_that_matches_no_areas_it_prints_a_warning(
self):
with CapturingIO('stdout') as stdout:
with self.assertRaises(SystemExit):
args = Namespace(uuid_or_alias='bogo-uuid')
ForgetCommand(args)
self.assertRegex(stdout.captured(), "don't recognize area")
def test_when_given_an_alias_that_matches_more_than_one_area_it_prints_a_warning(
self):
self.mock_upload_area('deadbeef-dead-dead-dead-beeeeeeeeeef')
self.mock_upload_area('deafbeef-deaf-deaf-deaf-beeeeeeeeeef')
with CapturingIO('stdout') as stdout:
with self.assertRaises(SystemExit):
args = Namespace(uuid_or_alias='dea')
ForgetCommand(args)
self.assertRegex(stdout.captured(), "matches more than one")
def test_upload_area_is_default_output_file_name(self, mock_check_file_statuses):
with CapturingIO('stdout') as stdout:
upload_area ='1234'
env = 'test'
args = Namespace(env=env, uuid=upload_area, output_file_name=None)
GenerateStatusReportCommand(args)
# upload area id used as file name if not passed in
mock_check_file_statuses.assert_called_once_with(upload_area, upload_area)
assert stdout.captured() == 'File status report for {}/{} generated, located {}.txt\n'.format(
env, upload_area, upload_area)
def test_cli_login(self):
"""Test that the login command works with a dummy token"""
access_token = "test_access_token"
expected = "Storing access credentials\n"
args = ["dss", "login", "--access-token", access_token]
with CapturingIO('stdout') as stdout:
hca.cli.main(args)
self.assertEqual(stdout.captured(), expected)
self.assertEqual(hca.get_config().oauth2_token.access_token, access_token)
def test_when_given_an_alias_that_matches_no_areas_it_prints_a_warning(self):
config = hca.get_config()
config.upload = {'areas': {}}
config.save()
with CapturingIO('stdout') as stdout:
args = Namespace(uri_or_alias='aaa')
SelectCommand(args)
six.assertRegex(self, stdout.captured(), "don't recognize area")
def _put_test_col(args, uuid=str(uuid.uuid4()), replica='aws'):
"""
Implements a context manager that PUTs a collection to the
data store using `hca dss put-collection` then deletes it
when done.
:param list args: arguments to pass to `hca dss put-collection`
:param str uuid: uuid of the collection
:param str replica: replica to use
:rtype: dict
:returns: put-collection response object
"""
base_args = ['dss', 'put-collection', '--replica', replica,
'--uuid', uuid]
with CapturingIO('stdout') as stdout:
hca.cli.main(args=base_args + args)
yield json.loads(stdout.captured())
base_args[1] = 'delete-collection'
with CapturingIO('stdout'):
hca.cli.main(args=base_args)
def test_creds(self):
area = self.mock_current_upload_area()
self.simulate_credentials_api(area_uuid=area.uuid)
with CapturingIO('stdout') as stdout:
args = Namespace(uuid_or_alias=area.uuid)
CredsCommand(args)
six.assertRegex(self, stdout.captured(), "AWS_ACCESS_KEY_ID=")
six.assertRegex(self, stdout.captured(), "AWS_SECRET_ACCESS_KEY=")
six.assertRegex(self, stdout.captured(), "AWS_SESSION_TOKEN=")
def test_list_area_command(self):
self.upload_bucket.Object('/'.join([self.area.uuid,
'file1.fastq.gz'])).put(Body="foo")
self.upload_bucket.Object('/'.join([self.area.uuid,
'sample.json'])).put(Body="foo")
self.simulate_credentials_api(area_uuid=self.area.uuid)
with CapturingIO('stdout') as stdout:
ListAreaCommand(Namespace(long=False))
self.assertEqual(stdout.captured(), "file1.fastq.gz\nsample.json\n")
def test_when_given_an_alias_that_matches_one_area_it_forgets_that_area(
self):
area = self.mock_current_upload_area()
self.assertIn(area.uuid, UploadConfig().areas)
self.assertEqual(area.uuid, UploadConfig().current_area)
with CapturingIO('stdout'):
args = Namespace(uuid_or_alias=area.uuid)
ForgetCommand(args)
self.assertNotIn(area.uuid, UploadConfig().areas)
self.assertEqual(None, UploadConfig().current_area)
def test_upload_progress_bar(self):
dirpath = os.path.join(TEST_DIR, 'tutorial', 'data') # arbitrary and small
put_args = ['dss', 'upload', '--src-dir', dirpath, '--replica',
'aws', '--staging-bucket', 'org-humancellatlas-dss-cli-test']
with self.subTest("Suppress progress bar if not interactive"):
with CapturingIO('stdout') as stdout:
hca.cli.main(args=put_args)
# If using CapturingIO, `hca dss upload` should know it's not being
# invoked interactively and as such not show a progress bar. Which
# means that stdout should parse nicely as json
self.assertTrue(json.loads(stdout.captured()))
def test_creds(self):
area = self.mock_current_upload_area()
self.simulate_credentials_api(area_uuid=area.uuid)
with CapturingIO('stdout') as stdout:
args = Namespace(uuid_or_alias=area.uuid)
CredsCommand(args)
non_blank_lines = [s for s in stdout.captured().split("\n") if s]
self.assertEqual(3, len(non_blank_lines))
self.assertRegex(stdout.captured(), "AWS_ACCESS_KEY_ID=")
self.assertRegex(stdout.captured(), "AWS_SECRET_ACCESS_KEY=")
self.assertRegex(stdout.captured(), "AWS_SESSION_TOKEN=")
def test_prints_file_status(self, mock_get_checksum_status):
mock_return_value = 'CHECKSUMMING'
filename = 'existing_file'
upload_area = '1234'
env = 'test'
mock_get_checksum_status.return_value = mock_return_value
with CapturingIO('stdout') as stdout:
args = Namespace(env=env, uuid=upload_area, filename=filename)
ListFileStatusCommand(args)
assert stdout.captured(
) == "File: {} in UploadArea: {}/{} is currently {}\n".format(
filename, env, upload_area, mock_return_value)
def test_use_selected_area_and_env_if_none_given(self,
mock_get_checksum_status):
mock_return_value = 'CHECKSUMMING'
mock_get_checksum_status.return_value = mock_return_value
filename = 'existing_file'
with CapturingIO('stdout') as stdout:
args = Namespace(env=None, uuid=None, filename=filename)
ListFileStatusCommand(args)
assert stdout.captured(
) == "File: {} in UploadArea: {}/{} is currently {}\n".format(
filename, 'test', self.area.uuid, mock_return_value)
def test_when_given_an_unrecognized_uri_without_slash_it_sets_it_as_current_area(
self):
uri_without_slash = "s3://org-humancellatlas-upload-test/{}".format(
self._area_uuid)
with CapturingIO('stdout'):
args = Namespace(uri_or_alias=uri_without_slash)
SelectCommand(args)
config = hca.get_config()
self.assertIn(self._area_uuid, config.upload.areas)
self.assertEqual(self._uri,
config.upload.areas[self._area_uuid]['uri'])
self.assertEqual(self._area_uuid, config.upload.current_area)
def test_remote_login(self):
"""Test that remote logins work for non-interactive systems
0. Change the skipIf from True to False to allow invocation of test
1. Follow the link provided by the test
2. Paste the code value into the test env
3. Confirm Results
"""
args = ["dss", "login", "--remote"]
hca.cli.main(args)
self.assertTrue(hca.get_config().oauth2_token.access_token)
args = ['dss', 'get-subscriptions', '--replica', 'aws']
with CapturingIO('stdout') as stdout:
hca.cli.main(args)
results = json.loads(stdout.captured())
self.assertIn("subscriptions", results)
def test_when_given_an_alias_that_matches_more_than_one_area_it_prints_a_warning(self):
config = hca.get_config()
config.upload = {
'areas': {
'deadbeef-dead-dead-dead-beeeeeeeeeef': {'uri': 's3://bucket/deadbeef-dead-dead-dead-beeeeeeeeeef/'},
'deafbeef-deaf-deaf-deaf-beeeeeeeeeef': {'uri': 's3://bucket/deafbeef-deaf-deaf-deaf-beeeeeeeeeef/'},
}
}
config.save()
with CapturingIO('stdout') as stdout:
args = Namespace(uri_or_alias='dea')
SelectCommand(args)
six.assertRegex(self, stdout.captured(), "matches more than one")
def test_get_files_cli(self):
with tempfile.TemporaryDirectory(prefix='cli-test-', suffix='.tmp',
dir=os.getcwd()) as dest_dir, \
self._put_test_bdl() as upload_res:
filename = "SRR2967608_1.fastq.gz"
file_path = os.path.join(TEST_DIR, 'res', 'bundle', filename)
download_args = ['dss', 'download', '--bundle-uuid', upload_res['bundle_uuid'],
'--replica', 'aws', '--download-dir', dest_dir]
with CapturingIO('stdout'):
hca.cli.main(args=download_args)
bundle_fqid = upload_res['bundle_uuid'] + '.' + upload_res['version']
with open(os.path.join(dest_dir, bundle_fqid, filename), 'rb') as download_data:
download_content = download_data.read()
with open(file_path, 'rb') as bytes_fh:
self.assertEqual(bytes_fh.read(), download_content)
def test_when_given_a_uri_it_prints_an_alias(self):
config = hca.get_config()
config.upload = {
'areas': {
'deadbeef-dead-dead-dead-beeeeeeeeeef': {
'uri': 's3://bogobucket/deadbeef-dead-dead-dead-beeeeeeeeeef/'
},
}
}
config.save()
new_area_uuid = 'deafbeef-deaf-deaf-deaf-beeeeeeeeeef'
new_area_uri = 's3://bogobucket/{}/'.format(new_area_uuid)
with CapturingIO('stdout') as stdout:
args = Namespace(uri_or_alias=new_area_uri)
SelectCommand(args)
six.assertRegex(self, stdout.captured(), "alias \"{}\"".format('deaf'))
def test_when_given_an_alias_that_matches_one_area_it_selects_it(self):
a_uuid = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'
b_uuid = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'
config = hca.get_config()
config.upload = {
'areas': {
a_uuid: {'uri': "s3://org-humancellatlas-upload-bogo/%s/" % (a_uuid,)},
b_uuid: {'uri': "s3://org-humancellatlas-upload-bogo/%s/" % (b_uuid,)},
}
}
config.save()
with CapturingIO('stdout') as stdout:
args = Namespace(uri_or_alias='bbb')
SelectCommand(args)
config = hca.get_config()
self.assertEqual(b_uuid, config.upload.current_area)
def test_it_lists_areas_when_there_are_some(self):
a_uuid = 'aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa'
b_uuid = 'bbbbbbbb-bbbb-bbbb-bbbb-bbbbbbbbbbbb'
config = hca.get_config()
config.upload = {
'areas': {
a_uuid: {
'uri': "s3://foo/{}/".format(a_uuid)
},
b_uuid: {
'uri': "s3://foo/{}/".format(b_uuid)
},
},
'current_area': a_uuid
}
config.save()
with CapturingIO('stdout') as stdout:
ListAreasCommand(Namespace())
six.assertRegex(self, stdout.captured(), "%s <- selected" % a_uuid)
six.assertRegex(self, stdout.captured(), b_uuid)
def test_version_output(self):
args = ["dss", "create-version"]
with CapturingIO('stdout') as stdout:
hca.cli.main(args=args)
print(stdout.captured())
self.assertTrue(stdout.captured())
