# folder_name = folder_names[0] for folder_name in folder_names: clean_folder_name = path_utils.clean_filename(folder_name) json_filename = f'{base_task_name}_{clean_folder_name}_all.json' list_file = os.path.join(filename_base, json_filename) # If this is intended to be a folder, it needs to end in '/', otherwise # files that start with the same string will match too folder_name = folder_name.replace('\\', '/') if len(folder_name) > 0 and (not folder_name.endswith('/')): folder_name = folder_name + '/' prefix = container_prefix + folder_name file_list = ai4e_azure_utils.enumerate_blobs_to_file( output_file=list_file, account_name=storage_account_name, container_name=container_name, sas_token=read_only_sas_token, blob_prefix=prefix) assert all(path_utils.is_image_file(s) for s in file_list) file_lists_by_folder.append(list_file) assert len(file_lists_by_folder) == len(folder_names) #%% Divide images into chunks for each folder # The JSON file at folder_chunks[i][j] corresponds to task j of taskgroup i folder_chunks = [] # list_file = file_lists_by_folder[0] for list_file in file_lists_by_folder: chunked_files, chunks = prepare_api_submission.divide_files_into_tasks(
#%% Interactive driver if False: #%% account_name = '' sas_token = 'st=...' container_name = '' rsearch = None # '^Y53' output_file = r'output.json' blobs = ai4e_azure_utils.enumerate_blobs_to_file( output_file=output_file, account_name=account_name, sas_token=sas_token, container_name=container_name, rsearch=rsearch) #%% file_list_json = r"D:\temp\idfg_20190801-hddrop_image_list.json" task_files = divide_files_into_tasks(file_list_json) #%% file_list_sas_urls = ['', '', ''] input_container_sas_url = '' request_name_base = '' caller = '*****@*****.**'
folder_name = folder_name + '/' prefixes = [container_prefix + folder_name] else: prefixes = [container_prefix + s for s in folder_prefixes[folder_name]] folder_images = [] for prefix in prefixes: print('Enumerating prefix {} for folder [{}]'.format(prefix,folder_name)) prefix_images = ai4e_azure_utils.enumerate_blobs_to_file( output_file=None, account_name=storage_account_name, container_name=container_name, sas_token=read_only_sas_token, blob_prefix=prefix, rsearch=None) folder_images.extend(prefix_images) # ...for each prefix print('Enumerated {} images for folder {}'.format(len(folder_images),folder_name)) # Write to file ai4e_azure_utils.write_list_to_file(list_file, folder_images) file_lists_by_folder.append(list_file) images_by_folder.append(folder_images) # ...for each folder