# folder_name = folder_names[0]
for folder_name in folder_names:
clean_folder_name = path_utils.clean_filename(folder_name)
json_filename = f'{base_task_name}_{clean_folder_name}_all.json'
list_file = os.path.join(filename_base, json_filename)
# If this is intended to be a folder, it needs to end in '/', otherwise
# files that start with the same string will match too
folder_name = folder_name.replace('\\', '/')
if len(folder_name) > 0 and (not folder_name.endswith('/')):
folder_name = folder_name + '/'
prefix = container_prefix + folder_name
file_list = ai4e_azure_utils.enumerate_blobs_to_file(
output_file=list_file,
account_name=storage_account_name,
container_name=container_name,
sas_token=read_only_sas_token,
blob_prefix=prefix)
assert all(path_utils.is_image_file(s) for s in file_list)
file_lists_by_folder.append(list_file)
assert len(file_lists_by_folder) == len(folder_names)
#%% Divide images into chunks for each folder
# The JSON file at folder_chunks[i][j] corresponds to task j of taskgroup i
folder_chunks = []
# list_file = file_lists_by_folder[0]
for list_file in file_lists_by_folder:
chunked_files, chunks = prepare_api_submission.divide_files_into_tasks(
#%% Interactive driver
if False:
#%%
account_name = ''
sas_token = 'st=...'
container_name = ''
rsearch = None # '^Y53'
output_file = r'output.json'
blobs = ai4e_azure_utils.enumerate_blobs_to_file(
output_file=output_file,
account_name=account_name,
sas_token=sas_token,
container_name=container_name,
rsearch=rsearch)
#%%
file_list_json = r"D:\temp\idfg_20190801-hddrop_image_list.json"
task_files = divide_files_into_tasks(file_list_json)
#%%
file_list_sas_urls = ['', '', '']
input_container_sas_url = ''
request_name_base = ''
caller = '*****@*****.**'
folder_name = folder_name + '/'
prefixes = [container_prefix + folder_name]
else:
prefixes = [container_prefix + s for s in folder_prefixes[folder_name]]
folder_images = []
for prefix in prefixes:
print('Enumerating prefix {} for folder [{}]'.format(prefix,folder_name))
prefix_images = ai4e_azure_utils.enumerate_blobs_to_file(
output_file=None,
account_name=storage_account_name,
container_name=container_name,
sas_token=read_only_sas_token,
blob_prefix=prefix,
rsearch=None)
folder_images.extend(prefix_images)
# ...for each prefix
print('Enumerated {} images for folder {}'.format(len(folder_images),folder_name))
# Write to file
ai4e_azure_utils.write_list_to_file(list_file, folder_images)
file_lists_by_folder.append(list_file)
images_by_folder.append(folder_images)
# ...for each folder
