def from_url(cls, url, authenticate=True, **kwargs): """Create the chunk store""" if not authenticate: return S3ChunkStore(url, timeout=TIMEOUT, retries=RETRY, **kwargs) if cls.httpd is None: proxy_host = '127.0.0.1' with get_free_port(proxy_host) as proxy_port: httpd = _TokenHTTPProxyServer((proxy_host, proxy_port), _TokenHTTPProxyHandler) httpd.target = url httpd.session = requests.Session() httpd.auth = _AWSAuth(cls.credentials) httpd.initial_request_time = {} cls.httpd_thread = threading.Thread(target=httpd.serve_forever) cls.httpd_thread.start() # We delay setting cls.httpd until we've launched serve_forever, # because teardown calls httpd.shutdown and that hangs if # serve_forever wasn't called. cls.httpd = httpd cls.proxy_url = 'http://{}:{}'.format(proxy_host, proxy_port) elif url != cls.httpd.target: raise RuntimeError( 'Cannot use multiple target URLs with http proxy') # The token only authorises the one known bucket token = encode_jwt({ 'alg': 'ES256', 'typ': 'JWT' }, {'prefix': [BUCKET]}) return S3ChunkStore(cls.proxy_url, timeout=TIMEOUT, retries=RETRY, token=token, **kwargs)
def test_store_unavailable_unresponsive_server(self): host = '127.0.0.1' with get_free_port(host) as port: url = f'http://{host}:{port}/' store = S3ChunkStore(url, timeout=0.1, retries=0) with assert_raises(StoreUnavailable): store.is_complete('store_is_not_listening_on_that_port')
def test_public_read(self): url, kwargs = self.prepare_store_args(self.s3_url, credentials=None) reader = S3ChunkStore(url, **kwargs) # Create a non-public-read array. # This test deliberately doesn't use array_name so that it can create # several different buckets. slices = np.index_exp[0:5] x = np.arange(5) self.store.create_array('private/x') self.store.put_chunk('private/x', slices, x) # Ceph RGW returns 403 for missing chunks too so we see ChunkNotFound with assert_raises(ChunkNotFound): reader.get_chunk('private/x', slices, x.dtype) # Now a public-read array url, kwargs = self.prepare_store_args(self.s3_url, public_read=True) store = S3ChunkStore(url, **kwargs) store.create_array('public/x') store.put_chunk('public/x', slices, x) y = reader.get_chunk('public/x', slices, x.dtype) np.testing.assert_array_equal(x, y)
def setup_class(cls): """Start minio service running on temp dir, and ChunkStore on that.""" cls.credentials = ('access*key', 'secret*key') cls.tempdir = tempfile.mkdtemp() cls.minio = None try: cls.s3_url = cls.start_minio('127.0.0.1') cls.store_url, cls.store_kwargs = cls.prepare_store_args( cls.s3_url) cls.store = S3ChunkStore(cls.store_url, **cls.store_kwargs) # Ensure that pagination is tested cls.store.list_max_keys = 3 except Exception: cls.teardown_class() raise
def setup_class(cls): """Start Fake S3 service running on temp dir, and ChunkStore on that.""" cls.tempdir = tempfile.mkdtemp() # XXX Python 3.3+ can use subprocess.DEVNULL instead cls.devnull = open(os.devnull, 'wb') cls.fakes3 = None cls.stderr_consumer = None try: url = cls.start_fakes3('127.0.0.1') try: cls.store = S3ChunkStore.from_url(url, timeout=1) except ImportError: raise SkipTest('S3 requests dependency not installed') # Ensure that pagination is tested # Disabled for now because FakeS3 doesn't implement it correctly # (see for example https://github.com/jubos/fake-s3/pull/163). # cls.store.list_max_keys = 3 except Exception: cls.teardown_class() raise
def test_token_without_https(self): # Don't allow users to leak their tokens by accident with assert_raises(StoreUnavailable): S3ChunkStore('http://apparently.invalid/', token='secrettoken')
max_dumps = args.max_dumps if args.max_dumps > 0 else vis.shape[0] use_rados = args.ceph_pool is not None if use_rados: obj_store = RadosChunkStore.from_config(args.ceph_conf, args.ceph_pool, args.ceph_keyring) pool_stats = obj_store.ioctx.get_stats() logger.info( "Connected to pool %s. Currently holds %d objects " "totalling %g GB", args.ceph_pool, pool_stats['num_objects'], pool_stats['num_bytes'] / 1e9) ts_pbs.add("ceph_pool", args.ceph_pool, immutable=True) with open(args.ceph_conf, "r") as ceph_conf: ts_pbs.add("ceph_conf", ceph_conf.readlines(), immutable=True) else: obj_store = S3ChunkStore.from_url(args.s3_url) ts_pbs.add("s3_endpoint", args.s3_url, immutable=True) target_object_size = args.obj_size * 2**20 dask_graph = {} schedule = dask.threaded.get output_keys = [] h5_store = DictChunkStore(**h5_file['Data']) for dataset, arr in h5_store.arrays.iteritems(): dataset = str(dataset) dtype = arr.dtype shape = arr.shape get = h5_store.get if dataset == 'correlator_data': # Convert from 2x float32 to complex64 (and swallow last dimension) dtype = np.dtype(np.complex64)
def from_url(cls, url, authenticate=True, **kwargs): """Create the chunk store""" if authenticate: kwargs['credentials'] = cls.credentials return S3ChunkStore(url, timeout=TIMEOUT, retries=RETRY, **kwargs)
def from_url(cls, url): """Create the chunk store""" return S3ChunkStore.from_url(url, timeout=1)
def from_url(cls, url): """Create the chunk store""" return S3ChunkStore.from_url(url, timeout=1, token='mysecret')