def test_object_exists(blobstore: BlobStore,
bucket: str,
match: str,
test_type: ObjectTest = ObjectTest.EXACT) -> bool:
"""
Test if an object exists in the BlobStore
:param blobstore: the blobstore to check
:param bucket: the bucket to check in the blobstore
:param match: the string to match against; this is _not_ a regex pattern, strings must match exactly
:param test_type: the type of test to conduct, prefix matches test if the object name starts with the match string,
exact matches must match the full string
:return: test bool
"""
if test_type == ObjectTest.PREFIX:
try:
blobstore.list(bucket, prefix=match).__iter__().__next__()
except StopIteration:
return False
else:
return True
elif test_type == ObjectTest.EXACT:
try:
blobstore.get_user_metadata(bucket, match)
return True
except BlobNotFoundError:
return False
else:
raise ValueError(f"Not a valid storage object test type: " +
test_type.name)
def map_bucket_results(func: typing.Callable,
handle: BlobStore,
bucket: str,
base_pfx: str,
parallelization=10):
"""
Call `func` on an iterable of keys
func is expected to be thread safe.
"""
with ThreadPoolExecutor(max_workers=parallelization) as e:
futures = list()
for pfx in "0123456789abcdef":
f = e.submit(func, handle.list(bucket, prefix=f"{base_pfx}{pfx}"))
futures.append(f)
for f in as_completed(futures):
try:
yield f.result()
except Exception:
logger.error(traceback.format_exc())