def get_configured_bucket():
with patch('quilt3.util.requests') as requests_mock:
FEDERATION_URL = 'https://test.com/federation.json'
mock_federation = {
'buckets': [{
'name': 'test-bucket',
'searchEndpoint': 'test'
}]
}
CONFIG_URL = 'https://test.com/config.json'
mock_config = {'federations': ['/federation.json']}
def makeResponse(text):
mock_response = ResponseMock()
setattr(mock_response, 'text', text)
setattr(mock_response, 'ok', True)
return mock_response
def mock_get(url):
if url == CONFIG_URL:
return makeResponse(json.dumps(mock_config))
elif url == FEDERATION_URL:
return makeResponse(json.dumps(mock_federation))
else:
raise Exception
requests_mock.get = mock_get
bucket = Bucket('s3://test-bucket')
bucket.config('https://test.com/config.json')
return bucket
def test_bucket_put_file(self):
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket = Bucket('s3://test-bucket')
bucket.put_file(key='README.md', path='./README') # put local file to bucket
copy_mock.assert_called_once_with(
PhysicalKey.from_path('README'), PhysicalKey.from_url('s3://test-bucket/README.md'))
def test_search_bucket(self, config_mock, search_mock):
config_mock.return_value = 'https://foo.bar'
content = {
'federations': ['/federation.json'],
}
federations = {
"buckets": [
{
"name": "quilt-testing-fake",
"searchEndpoint": "https://es-fake.endpoint",
"region": "us-meow"
},
]
}
self.requests_mock.add(responses.GET,
'https://foo.bar/config.json',
json=content,
status=200)
self.requests_mock.add(responses.GET,
'https://foo.bar/federation.json',
json=federations,
status=200)
b = Bucket('s3://quilt-testing-fake')
b.search('blah', limit=1)
config_mock.assert_called_once_with('navigator_url')
search_mock.assert_called_once_with('blah',
'https://es-fake.endpoint',
limit=1,
aws_region='us-meow',
bucket='quilt-testing-fake')
def test_bucket_meta(self):
test_meta = {'helium': json.dumps({'target': 'json'})}
response = {'Metadata': test_meta, 'ContentLength': 123}
params = {'Bucket': 'test-bucket', 'Key': 'test'}
self.s3_stubber.add_response('head_object', response, params)
bucket = Bucket('s3://test-bucket')
meta = bucket.get_meta('test')
assert meta == {'target': 'json'}
head_meta = {'helium': json.dumps({"target": "json"})}
head_response = {'Metadata': head_meta, 'ContentLength': 123}
head_params = {'Bucket': 'test-bucket', 'Key': 'test'}
self.s3_stubber.add_response('head_object', head_response, head_params)
new_test_meta = {
'helium': json.dumps({
'target': 'json',
'user_meta': {}
})
}
response = {}
params = {
'CopySource': {
'Bucket': 'test-bucket',
'Key': 'test'
},
'Bucket': 'test-bucket',
'Key': 'test',
'Metadata': new_test_meta,
'MetadataDirective': 'REPLACE'
}
self.s3_stubber.add_response('copy_object', response, params)
bucket.set_meta('test', {})
def test_bucket_select(self):
# Stubber doesn't have an accurate shape for the results of select_object_content
chunks = [
b'{"foo": ',
b'9, "b',
b'ar": 3',
b'}\n{"foo"',
b': 9, "bar": 1}\n{"foo": 6, "bar": 9}\n{"foo":',
b' 1, "bar": 7}\n{"foo":',
b' 6, "bar": 1}\n{"foo": 6, "bar": 6}',
b'\n{"foo": 9, "bar": 6}',
b'\n{"foo": 6, "bar": 4}\n',
b'{"foo": 2, "bar": 0}',
b'\n{"foo": 2, "bar": 0}\n',
]
records = [{'Records': {'Payload': chunk}} for chunk in chunks]
# noinspection PyTypeChecker
records.append({'Stats': {
'BytesScanned': 100,
'BytesProcessed': 100,
'BytesReturned': 210,
}})
records.append({'End': {}})
expected_result = pd.DataFrame.from_records([
{'foo': 9, 'bar': 3},
{'foo': 9, 'bar': 1},
{'foo': 6, 'bar': 9},
{'foo': 1, 'bar': 7},
{'foo': 6, 'bar': 1},
{'foo': 6, 'bar': 6},
{'foo': 9, 'bar': 6},
{'foo': 6, 'bar': 4},
{'foo': 2, 'bar': 0},
{'foo': 2, 'bar': 0},
])
# test normal use from extension
expected_args = {
'Bucket': 'test-bucket',
'Key': 'test.json',
'Expression': 'select * from S3Object',
'ExpressionType': 'SQL',
'InputSerialization': {
'CompressionType': 'NONE',
'JSON': {'Type': 'DOCUMENT'}
},
'OutputSerialization': {'JSON': {}},
}
boto_return_val = {'Payload': iter(records)}
with patch.object(self.s3_client, 'select_object_content', return_value=boto_return_val) as patched:
bucket = Bucket('s3://test-bucket')
result = bucket.select('test.json', 'select * from S3Object')
patched.assert_called_once_with(**expected_args)
assert result.equals(expected_result)
def test_bucket_put_file(self):
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket = Bucket('s3://test-bucket')
bucket.put_file(key='README.md',
path='./README') # put local file to bucket
copy_src = copy_mock.call_args_list[0][0][0]
assert urlparse(copy_src).scheme == 'file'
copy_dest = copy_mock.call_args_list[0][0][1]
assert urlparse(copy_dest).scheme == 's3'
def test_remote_delete_dir(self):
self.s3_stubber.add_response(
method='list_objects_v2',
service_response={
'IsTruncated': False,
'Contents': [{'Key': 'dir/a'}, {'Key': 'dir/b'}],
},
expected_params={
'Bucket': 'test-bucket',
'Prefix': 'dir/'
}
)
self.s3_stubber.add_response(
method='head_object',
service_response={},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/a'
}
)
self.s3_stubber.add_response(
method='delete_object',
service_response={},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/a'
}
)
self.s3_stubber.add_response(
method='head_object',
service_response={},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/b'
}
)
self.s3_stubber.add_response(
method='delete_object',
service_response={},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/b'
}
)
bucket = Bucket('s3://test-bucket')
bucket.delete_dir('dir/')
with pytest.raises(ValueError):
bucket.delete_dir('dir')
def test_bucket_put_file(self):
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket = Bucket('s3://test-bucket')
bucket.put_file(key='README.md',
path='./README') # put local file to bucket
copy_src = copy_mock.call_args_list[0][0][0]
assert urlparse(copy_src).scheme == 'file'
copy_dest = copy_mock.call_args_list[0][0][1]
assert urlparse(copy_dest).scheme == 's3'
copy_mock.reset_mock()
test_meta = {'asdf': 'jkl;'}
expected_meta = {'user_meta': test_meta}
bucket.put_file(key='README.md', path='./README', meta=test_meta)
(src, dest, meta) = copy_mock.call_args_list[0][0]
assert meta == expected_meta
def test_bucket_put_dir(self):
path = pathlib.Path(__file__).parent / 'data'
bucket = Bucket('s3://test-bucket')
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket.put_dir('test', path)
copy_mock.assert_called_once_with(path.as_uri() + '/',
's3://test-bucket/test/')
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket.put_dir('test/', path)
copy_mock.assert_called_once_with(path.as_uri() + '/',
's3://test-bucket/test/')
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket.put_dir('', path)
copy_mock.assert_called_once_with(path.as_uri() + '/',
's3://test-bucket/')
def test_bucket_put_dir(self):
path = pathlib.Path(__file__).parent / 'data'
bucket = Bucket('s3://test-bucket')
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket.put_dir('test', path)
copy_mock.assert_called_once_with(
PhysicalKey.from_path(str(path) + '/'), PhysicalKey.from_url('s3://test-bucket/test/'))
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket.put_dir('test/', path)
copy_mock.assert_called_once_with(
PhysicalKey.from_path(str(path) + '/'), PhysicalKey.from_url('s3://test-bucket/test/'))
with patch("quilt3.bucket.copy_file") as copy_mock:
bucket.put_dir('', path)
copy_mock.assert_called_once_with(
PhysicalKey.from_path(str(path) + '/'), PhysicalKey.from_url('s3://test-bucket/'))
def test_bucket_put_ext(self, put_bytes):
# This just ensures the bucket is calling serialize() correctly.
obj = 'just a string..'
b = Bucket('s3://quilt-testing-fake')
b.put('foo.json', obj)
assert put_bytes.called
assert len(put_bytes.call_args_list) == 1
args, kwargs = put_bytes.call_args
# avoid args[n] call if put_bytes was called w/kwarg arguments
data = kwargs['data'] if 'data' in kwargs else args[0]
dest = kwargs['dest'] if 'dest' in kwargs else args[1]
meta = kwargs['meta'] if 'meta' in kwargs else args[2]
assert json.loads(data) == obj
assert dest == 's3://quilt-testing-fake/foo.json'
assert meta.get('format', {}).get('name') == 'json'
def test_remote_delete(self):
self.s3_stubber.add_response(method='head_object',
service_response={},
expected_params={
'Bucket': 'test-bucket',
'Key': 'file.json',
})
self.s3_stubber.add_response(method='delete_object',
service_response={},
expected_params={
'Bucket': 'test-bucket',
'Key': 'file.json',
})
bucket = Bucket('s3://test-bucket')
bucket.delete('file.json')
with pytest.raises(QuiltException):
bucket.delete('dir/')
def test_bucket_config(self, config_mock, bucket_config_mock):
bucket_config_mock.return_value = {
'name': 'test-bucket',
'title': 'Test Bucket',
'icon': 'url',
'description': 'description',
'searchEndpoint': 'https://foo.bar/search'
}
config_mock.return_value = 'https://foo.bar'
b = Bucket('s3://test-bucket')
b.config()
assert b._search_endpoint == 'https://foo.bar/search'
config_mock.assert_called_once_with('navigator_url')
bucket_config_mock.assert_called_once_with(
'test-bucket', 'https://foo.bar/config.json')
config_mock.reset_mock()
bucket_config_mock.reset_mock()
b.config('https://bar.foo/config.json')
assert not config_mock.called
bucket_config_mock.assert_called_once_with(
'test-bucket', 'https://bar.foo/config.json')
def test_bucket_fetch(self):
bucket = Bucket('s3://test-bucket')
a_contents = b'a' * 10
b_contents = b'b' * 20
# Fetch a directory.
self.s3_stubber.add_response(method='list_objects_v2',
service_response={
'IsTruncated':
False,
'Contents': [{
'Key': 'dir/a',
'Size': len(a_contents)
}, {
'Key': 'dir/foo/b',
'Size': len(b_contents)
}],
},
expected_params={
'Bucket': 'test-bucket',
'Prefix': 'dir/'
})
self.s3_stubber.add_response(method='get_object',
service_response={
'ContentLength': len(a_contents),
'Body': BytesIO(a_contents)
},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/a'
})
self.s3_stubber.add_response(method='get_object',
service_response={
'ContentLength': len(b_contents),
'Body': BytesIO(b_contents)
},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/foo/b'
})
with patch(
'quilt3.data_transfer.s3_transfer_config.max_request_concurrency',
1):
bucket.fetch('dir/', './')
assert pathlib.Path('a').read_bytes() == a_contents
assert pathlib.Path('foo/b').read_bytes() == b_contents
# Fetch a single file.
self.s3_stubber.add_response(method='head_object',
service_response={
'ContentLength': len(b_contents),
},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/foo/b'
})
self.s3_stubber.add_response(method='get_object',
service_response={
'ContentLength': len(b_contents),
'Body': BytesIO(b_contents)
},
expected_params={
'Bucket': 'test-bucket',
'Key': 'dir/foo/b'
})
bucket.fetch('dir/foo/b', './blah/')
assert pathlib.Path('blah/b').read_bytes() == b_contents
# Fetch a non-existent directory.
self.s3_stubber.add_response(method='list_objects_v2',
service_response={'IsTruncated': False},
expected_params={
'Bucket': 'test-bucket',
'Prefix': 'does/not/exist/'
})
with pytest.raises(QuiltException):
bucket.fetch('does/not/exist/', './')
def test_bucket_construct(self):
Bucket('s3://test-bucket')
def test_bucket_fetch(self):
response = {'IsTruncated': False}
params = {'Bucket': 'test-bucket', 'Prefix': 'does/not/exist/'}
self.s3_stubber.add_response('list_objects_v2', response, params)
with pytest.raises(QuiltException):
Bucket('s3://test-bucket').fetch('does/not/exist/', './')
