def from_url(cls, url, authenticate=True, **kwargs):
"""Create the chunk store"""
if not authenticate:
return S3ChunkStore(url, timeout=TIMEOUT, retries=RETRY, **kwargs)
if cls.httpd is None:
proxy_host = '127.0.0.1'
with get_free_port(proxy_host) as proxy_port:
httpd = _TokenHTTPProxyServer((proxy_host, proxy_port),
_TokenHTTPProxyHandler)
httpd.target = url
httpd.session = requests.Session()
httpd.auth = _AWSAuth(cls.credentials)
httpd.initial_request_time = {}
cls.httpd_thread = threading.Thread(target=httpd.serve_forever)
cls.httpd_thread.start()
# We delay setting cls.httpd until we've launched serve_forever,
# because teardown calls httpd.shutdown and that hangs if
# serve_forever wasn't called.
cls.httpd = httpd
cls.proxy_url = 'http://{}:{}'.format(proxy_host, proxy_port)
elif url != cls.httpd.target:
raise RuntimeError(
'Cannot use multiple target URLs with http proxy')
# The token only authorises the one known bucket
token = encode_jwt({
'alg': 'ES256',
'typ': 'JWT'
}, {'prefix': [BUCKET]})
return S3ChunkStore(cls.proxy_url,
timeout=TIMEOUT,
retries=RETRY,
token=token,
**kwargs)
def test_store_unavailable_unresponsive_server(self):
host = '127.0.0.1'
with get_free_port(host) as port:
url = f'http://{host}:{port}/'
store = S3ChunkStore(url, timeout=0.1, retries=0)
with assert_raises(StoreUnavailable):
store.is_complete('store_is_not_listening_on_that_port')
def test_public_read(self):
url, kwargs = self.prepare_store_args(self.s3_url, credentials=None)
reader = S3ChunkStore(url, **kwargs)
# Create a non-public-read array.
# This test deliberately doesn't use array_name so that it can create
# several different buckets.
slices = np.index_exp[0:5]
x = np.arange(5)
self.store.create_array('private/x')
self.store.put_chunk('private/x', slices, x)
# Ceph RGW returns 403 for missing chunks too so we see ChunkNotFound
with assert_raises(ChunkNotFound):
reader.get_chunk('private/x', slices, x.dtype)
# Now a public-read array
url, kwargs = self.prepare_store_args(self.s3_url, public_read=True)
store = S3ChunkStore(url, **kwargs)
store.create_array('public/x')
store.put_chunk('public/x', slices, x)
y = reader.get_chunk('public/x', slices, x.dtype)
np.testing.assert_array_equal(x, y)
def setup_class(cls):
"""Start minio service running on temp dir, and ChunkStore on that."""
cls.credentials = ('access*key', 'secret*key')
cls.tempdir = tempfile.mkdtemp()
cls.minio = None
try:
cls.s3_url = cls.start_minio('127.0.0.1')
cls.store_url, cls.store_kwargs = cls.prepare_store_args(
cls.s3_url)
cls.store = S3ChunkStore(cls.store_url, **cls.store_kwargs)
# Ensure that pagination is tested
cls.store.list_max_keys = 3
except Exception:
cls.teardown_class()
raise
def setup_class(cls):
"""Start Fake S3 service running on temp dir, and ChunkStore on that."""
cls.tempdir = tempfile.mkdtemp()
# XXX Python 3.3+ can use subprocess.DEVNULL instead
cls.devnull = open(os.devnull, 'wb')
cls.fakes3 = None
cls.stderr_consumer = None
try:
url = cls.start_fakes3('127.0.0.1')
try:
cls.store = S3ChunkStore.from_url(url, timeout=1)
except ImportError:
raise SkipTest('S3 requests dependency not installed')
# Ensure that pagination is tested
# Disabled for now because FakeS3 doesn't implement it correctly
# (see for example https://github.com/jubos/fake-s3/pull/163).
# cls.store.list_max_keys = 3
except Exception:
cls.teardown_class()
raise
def test_token_without_https(self):
# Don't allow users to leak their tokens by accident
with assert_raises(StoreUnavailable):
S3ChunkStore('http://apparently.invalid/', token='secrettoken')
max_dumps = args.max_dumps if args.max_dumps > 0 else vis.shape[0]
use_rados = args.ceph_pool is not None
if use_rados:
obj_store = RadosChunkStore.from_config(args.ceph_conf, args.ceph_pool,
args.ceph_keyring)
pool_stats = obj_store.ioctx.get_stats()
logger.info(
"Connected to pool %s. Currently holds %d objects "
"totalling %g GB", args.ceph_pool, pool_stats['num_objects'],
pool_stats['num_bytes'] / 1e9)
ts_pbs.add("ceph_pool", args.ceph_pool, immutable=True)
with open(args.ceph_conf, "r") as ceph_conf:
ts_pbs.add("ceph_conf", ceph_conf.readlines(), immutable=True)
else:
obj_store = S3ChunkStore.from_url(args.s3_url)
ts_pbs.add("s3_endpoint", args.s3_url, immutable=True)
target_object_size = args.obj_size * 2**20
dask_graph = {}
schedule = dask.threaded.get
output_keys = []
h5_store = DictChunkStore(**h5_file['Data'])
for dataset, arr in h5_store.arrays.iteritems():
dataset = str(dataset)
dtype = arr.dtype
shape = arr.shape
get = h5_store.get
if dataset == 'correlator_data':
# Convert from 2x float32 to complex64 (and swallow last dimension)
dtype = np.dtype(np.complex64)
def from_url(cls, url, authenticate=True, **kwargs):
"""Create the chunk store"""
if authenticate:
kwargs['credentials'] = cls.credentials
return S3ChunkStore(url, timeout=TIMEOUT, retries=RETRY, **kwargs)
def from_url(cls, url):
"""Create the chunk store"""
return S3ChunkStore.from_url(url, timeout=1)
def from_url(cls, url):
"""Create the chunk store"""
return S3ChunkStore.from_url(url, timeout=1, token='mysecret')