def setup_class(cls):
"""Start Fake S3 service running on temp dir, and ChunkStore on that."""
cls.tempdir = tempfile.mkdtemp()
# XXX Python 3.3+ can use subprocess.DEVNULL instead
cls.devnull = open(os.devnull, 'wb')
cls.fakes3 = None
cls.stderr_consumer = None
try:
url = cls.start_fakes3('127.0.0.1')
try:
cls.store = S3ChunkStore.from_url(url, timeout=1)
except ImportError:
raise SkipTest('S3 requests dependency not installed')
# Ensure that pagination is tested
# Disabled for now because FakeS3 doesn't implement it correctly
# (see for example https://github.com/jubos/fake-s3/pull/163).
# cls.store.list_max_keys = 3
except Exception:
cls.teardown_class()
raise
max_dumps = args.max_dumps if args.max_dumps > 0 else vis.shape[0]
use_rados = args.ceph_pool is not None
if use_rados:
obj_store = RadosChunkStore.from_config(args.ceph_conf, args.ceph_pool,
args.ceph_keyring)
pool_stats = obj_store.ioctx.get_stats()
logger.info(
"Connected to pool %s. Currently holds %d objects "
"totalling %g GB", args.ceph_pool, pool_stats['num_objects'],
pool_stats['num_bytes'] / 1e9)
ts_pbs.add("ceph_pool", args.ceph_pool, immutable=True)
with open(args.ceph_conf, "r") as ceph_conf:
ts_pbs.add("ceph_conf", ceph_conf.readlines(), immutable=True)
else:
obj_store = S3ChunkStore.from_url(args.s3_url)
ts_pbs.add("s3_endpoint", args.s3_url, immutable=True)
target_object_size = args.obj_size * 2**20
dask_graph = {}
schedule = dask.threaded.get
output_keys = []
h5_store = DictChunkStore(**h5_file['Data'])
for dataset, arr in h5_store.arrays.iteritems():
dataset = str(dataset)
dtype = arr.dtype
shape = arr.shape
get = h5_store.get
if dataset == 'correlator_data':
# Convert from 2x float32 to complex64 (and swallow last dimension)
dtype = np.dtype(np.complex64)
def from_url(cls, url):
"""Create the chunk store"""
return S3ChunkStore.from_url(url, timeout=1)
def from_url(cls, url):
"""Create the chunk store"""
return S3ChunkStore.from_url(url, timeout=1, token='mysecret')