def parse_input_path_for_lithops(sm_config, input_path):
if input_path.startswith('s3://') or input_path.startswith('s3a://'):
backend = 'aws_s3'
bucket, prefix = split_s3_path(input_path)
else:
backend = 'ibm_cos'
bucket, prefix = split_cos_path(input_path)
storage = Storage(sm_config['lithops'], backend)
if backend == 'aws_s3' and sm_config['lithops']['aws_s3'][
'endpoint'].startswith('http://'):
# WORKAROUND for local Minio access
# Lithops forces the url to HTTPS, so overwrite the S3 client with a fixed client
# https://github.com/lithops-cloud/lithops/issues/708
storage.storage_handler.s3_client = get_s3_client()
keys_in_path = storage.list_keys(bucket, prefix)
imzml_keys = [
key for key in keys_in_path if key.lower().endswith('.imzml')
]
ibd_keys = [key for key in keys_in_path if key.lower().endswith('.ibd')]
debug_info = f'Path {input_path} had keys: {keys_in_path}'
assert len(
imzml_keys) == 1, f'Couldn\'t determine imzML file. {debug_info}'
assert len(ibd_keys) == 1, f'Couldn\'t determine ibd file. {debug_info}'
imzml_cobject = CloudObject(storage.backend, bucket, imzml_keys[0])
ibd_cobject = CloudObject(storage.backend, bucket, ibd_keys[0])
return storage, imzml_cobject, ibd_cobject
def validate_summaries(key_prefix, bucket_name, storage):
storage_client = Storage()
key_list = storage_client.list_keys(bucket_name, key_prefix + '/')
sorted_key_list = sorted(key_list, key=lambda x: int(x.split('/')[-1]))
summaries_buf = io.BytesIO()
# Get all summaries into one buffer
for key_name in sorted_key_list:
with open(f's3://{storage.bucket}/{key_name}',
'rb',
transport_params=dict(
client=storage.get_client())) as source_file:
copyfileobj(source_file, summaries_buf)
cmd = ['./valsort', '-s', '/dev/stdin']
with subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stdin=subprocess.PIPE,
stderr=subprocess.PIPE) as p:
with p.stdin as valinput, p.stdout as valoutput, p.stderr as valerr:
with p.stdin as valinput: # Need to close input for valsort to finish
valinput.write(summaries_buf.getbuffer())
returncode = p.wait()
if returncode != 0:
raise Exception(
f'Non-zero return code for valsort: {returncode}\n' +
valerr.read().decode('utf-8'))
valoutput_str = valoutput.read().decode('utf-8')
return valoutput_str
def validate_command(prefix, image):
storage_client = Storage()
with FunctionExecutor(runtime=image) as fexec:
bucket = fexec.config['lithops']['storage_bucket']
key_list = storage_client.list_keys(bucket, prefix + '/')
validate_records_futures = fexec.map(validate_records,
key_list,
extra_args=[bucket, prefix],
include_modules=['util'])
results = fexec.get_result(fs=validate_records_futures)
for index, r in enumerate(results):
if not r['success']:
print(f'Failed to validate partition: {key_list[index]}')
print(r['stderr'])
return
validate_summaries_futures = fexec.map(validate_summaries,
[prefix + summary_postfix],
extra_args=[bucket],
include_modules=['util'])
results = fexec.get_result(fs=validate_summaries_futures)
if results[0] == '':
print('Success!')
else:
print(results)
def empty_bucket(bucket, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
logger.info('Deleting all objects in bucket "{}"'.format(bucket))
keys = storage.list_keys(bucket)
logger.info('Total objects found: {}'.format(len(keys)))
storage.delete_objects(bucket, keys)
logger.info('All objects deleted successfully')
def delete_object(bucket, key, prefix, backend, debug):
log_level = logging.INFO if not debug else logging.DEBUG
setup_lithops_logger(log_level)
storage = Storage(backend=backend)
if key:
logger.info('Deleting object "{}" from bucket "{}"'.format(key, bucket))
storage.delete_object(bucket, key)
logger.info('Object deleted successfully')
elif prefix:
objs = storage.list_keys(bucket, prefix)
logger.info('Deleting {} objects with prefix "{}" from bucket "{}"'.format(len(objs), prefix, bucket))
storage.delete_objects(bucket, objs)
logger.info('Object deleted successfully')
def sort_command(input_prefix, output_prefix, max_parallelism, image):
storage_client = Storage()
bucket = None
input_info_lis = None
with FunctionExecutor(runtime=image, workers=max_parallelism) as fexec:
bucket = fexec.config['lithops']['storage_bucket']
input_info_list = storage_client.list_objects(bucket,
input_prefix + '/')
input_size = sum(info['Size'] for info in input_info_list)
(num_shuffles, last_values_per_category) = make_plan(input_size)
current_values_per_category = 1
current_prefix = input_prefix
current_keys_list = [{
'keys_list': [key_name],
'prefix': input_prefix + '-intermediate0',
'category_stack': []
} for key_name in storage_client.list_keys(bucket, input_prefix + '/')]
for current_shuffle in range(num_shuffles):
# Change values per category of last shuffle
if current_shuffle == num_shuffles - 1:
current_values_per_category = last_values_per_category
radix_sort_futures = fexec.map(radix_sort_by_byte,
current_keys_list,
extra_args={
'values_per_category':
current_values_per_category
},
include_modules=['util'])
radix_sort_results = fexec.get_result(fs=radix_sort_futures)
categories_keys_lists = {}
for res in radix_sort_results:
intermediate_keys_list = res['keys_list']
input_category_stack = res['category_stack']
for key_name in intermediate_keys_list:
category_id = int(key_name.rsplit(sep='/', maxsplit=3)[-3])
new_category_stack = input_category_stack + [category_id]
new_category_stack_str = '/'.join(
[str(x) for x in new_category_stack])
if new_category_stack_str in categories_keys_lists:
categories_keys_lists[new_category_stack_str].append(
key_name)
else:
categories_keys_lists[new_category_stack_str] = [
key_name
]
# Partition category lists
# Attach prefix metadata so that sorter knows what to name files
each_category_size = input_size / (
(256 / current_values_per_category) * (current_shuffle + 1))
num_partitions_per_category = math.ceil(each_category_size /
buffer_size_to_categorize)
current_keys_list = []
for category_stack_str, cat_keys_list in categories_keys_lists.items(
):
for sub_list in np.array_split(cat_keys_list,
num_partitions_per_category):
partition_entry = {
'keys_list':
sub_list,
'prefix':
f'{input_prefix}-intermediate{str(current_shuffle + 1)}',
'category_stack':
[int(x) for x in category_stack_str.split('/')]
}
current_keys_list.append(partition_entry)
consider_last_byte_sorted = False
if last_values_per_category == 1:
consider_last_byte_sorted = True
for entry in current_keys_list:
entry['prefix'] = output_prefix
sorted_keys_list = sorted(current_keys_list,
key=lambda x: x['category_stack'])
sort_category_futures = fexec.map(sort_category,
sorted_keys_list,
extra_args={
'consider_last_byte_sorted':
consider_last_byte_sorted
},
include_modules=['util'])
results = fexec.get_result(fs=sort_category_futures)
# print(results)
# Check if size of output matches size of input
output_info_list = storage_client.list_objects(bucket, output_prefix)
output_size = sum(info['Size'] for info in output_info_list)
assert input_size == output_size, f'input size: {input_size}, output_size: {output_size}'
print('Done!')
