def get_flattened_files_info(source_dir: str) -> [(str, str)]:
"""
:param source_dir: Path of the directory containing different versions
of operator bundles (CRD, CSV, package) in separate version directories
:return: A list of tuples where in each tuple, the first element is
the source file path to be copied to the flattened directory, and the second is
the new file name to be used in the file copy.
"""
# get package content and check if CSV exists in source_dir root, and
# process all subdirectories and filter those that do not contain valid manifest files
root_path, dir_names, root_dir_files = next(os.walk(source_dir))
csvs_path_and_content, pkg_path_and_content \
= get_csvs_pkg_info_from_root(source_dir)
dir_paths = [os.path.join(source_dir, dir_name) for dir_name in dir_names]
manifest_paths = list(filter(lambda x: is_manifest_folder(x), dir_paths))
# nested layout
if manifest_paths:
file_paths_to_copy = [] # [ (SRC_FILE_PATH, NEW_FILE_NAME) ]
crd_dict = {} # { CRD_NAME => (VERSION, CRD_PATH) }
csv_paths = []
for manifest_path in manifest_paths:
folder_semver = get_folder_semver(manifest_path)
if not folder_semver:
continue
parse_manifest_folder(manifest_path, folder_semver, csv_paths,
crd_dict)
# add package in source_dir
package_path = pkg_path_and_content[0]
file_paths_to_copy.append(
(package_path, os.path.basename(package_path)))
# add all CRDs with the latest version
for _, crd_path in crd_dict.values():
file_paths_to_copy.append((crd_path, os.path.basename(crd_path)))
# add all CSVs
for csv_file_name, csv_entries in create_csv_dict(csv_paths).items():
for (version, csv_path) in csv_entries:
basename, ext = os.path.splitext(csv_file_name)
file_paths_to_copy.append(
(csv_path, f'{basename}-v{version}{ext}'))
return file_paths_to_copy
# flat layout
elif pkg_path_and_content and csvs_path_and_content:
logger.info('The source directory is already flat.')
# just return files from dir as they are already flat
return [(os.path.join(source_dir, name), name)
for name in root_dir_files]
msg = 'The source directory structure is not in valid flat or nested format,' \
'because no valid CSV file is found in root or manifest directories.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
def get_manifests_info(self, source_dir):
"""
Given a source directory, this method returns a dict containing all
operator manifest file information, grouped by subfolder name if the
manifest directory structure is nested.
:param source_dir: Path to local directory of operator manifests, which can be
in either flat or nested format
:return: A dictionary object where the key is the folder name of the operator
manifest, and the value is a list of yaml strings of manifest files
FLAT_KEY is used as key if the directory structure is flat
"""
# MANIFEST_DIR_NAME => manifest_files_content
# FLAT_KEY is used as key to indicate the flat directory structure
manifests = {}
root_path, dir_names, root_dir_files = next(os.walk(source_dir))
csvs_path_and_content, pkg_path_and_content \
= get_csvs_pkg_info_from_root(source_dir)
dir_paths = [
os.path.join(source_dir, dir_name) for dir_name in dir_names
]
manifest_paths = list(
filter(lambda x: is_manifest_folder(x), dir_paths))
# if there is at least 1 valid manifest folder, we treat the directory layout as
# nested, and add all manifest files from each version folder to manifests dict
# nested layout: add package to each manifest dict entry
if manifest_paths:
logger.info('The source directory is in nested structure.')
self.nested = True
for manifest_path in manifest_paths:
manifest_dir_name = os.path.basename(manifest_path)
crd_files_info, csv_files_info = get_crd_csv_files_info(
manifest_path)
manifests[manifest_dir_name] = crd_files_info + csv_files_info
for manifest_dir_name in manifests:
manifests[manifest_dir_name].append(pkg_path_and_content)
# flat layout: collect all valid manifest files and add to FLAT_KEY entry
elif pkg_path_and_content and csvs_path_and_content:
logger.info('The source directory is in flat structure.')
crd_files_info, csv_files_info = get_crd_csv_files_info(root_path)
files_info = [pkg_path_and_content]
files_info.extend(crd_files_info + csv_files_info)
manifests[FLAT_KEY] = files_info
else:
msg = 'The source directory structure is not in valid flat or nested format,'\
'because no valid CSV file is found in root or manifest directories.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
return manifests
def nest_bundles(source_dir, output_dir):
root_path, dir_names, root_dir_files = next(os.walk(source_dir))
csvs_info, pkg_info = get_csvs_pkg_info_from_root(source_dir)
dir_paths = [os.path.join(source_dir, dir_name) for dir_name in dir_names]
manifest_paths = list(filter(lambda x: is_manifest_folder(x), dir_paths))
# nested layout
if manifest_paths:
logger.warning('The source directory is already nested.')
# extract paths of package file in root dir and
# valid CRD/CSV files from subdirectories, and ignore irrelevant ones
manifest_files_path = [pkg_info[0]]
for manifest_path in manifest_paths:
crds_info, csvs_info = get_crd_csv_files_info(manifest_path)
crd_csv_file_paths = [
file_info[0] for file_info in (crds_info + csvs_info)
]
manifest_files_path.extend(crd_csv_file_paths)
# copy all manifest files to output_dir with folder structure preserved
os.makedirs(output_dir, exist_ok=True)
for file_path in manifest_files_path:
file_path_relative = os.path.relpath(file_path, source_dir)
output_file_path = os.path.join(output_dir, file_path_relative)
os.makedirs(os.path.dirname(output_file_path), exist_ok=True)
copyfile(file_path, output_file_path)
# flat layout
elif csvs_info and pkg_info:
# extract all valid manifest (CRD, CSV, PKG) files from root
# and make nested bundles
crds_info, csvs_info = get_crd_csv_files_info(source_dir)
with TemporaryDirectory() as temp_dir:
manifest_files_content = [pkg_info[1]]
manifest_files_content.extend([
file_content for (_, file_content) in (crds_info + csvs_info)
])
nest_flat_bundles(manifest_files_content, output_dir, temp_dir)
else:
msg = 'The source directory structure is not in valid flat or nested format,' \
'because no valid CSV file is found in root or manifest directories.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})