def test_get_operator_artifact_type_assertions(fname):
with open(fname) as f:
yaml = f.read()
with pytest.raises(OpCourierBadArtifact) as e, LogCapture() as logs:
identify.get_operator_artifact_type(yaml)
logs.check(('operatorcourier.identify', 'ERROR',
'Courier requires valid CSV, CRD, and Package files'), )
assert 'Courier requires valid CSV, CRD, and Package files' == str(e.value)
def test_get_operator_artifact_type_with_invalid_yaml(fname):
with open(fname) as f:
yaml = f.read()
with pytest.raises(OpCourierBadYaml) as e, LogCapture() as logs:
identify.get_operator_artifact_type(yaml)
logs.check(('operatorcourier.identify', 'ERROR',
'Courier requires valid input YAML files'), )
assert 'Courier requires valid input YAML files' == str(e.value)
def get_package_path(base_dir: str, file_names_in_base_dir: list) -> str:
packages = []
# add package file to file_paths_to_copy
# only 1 package yaml file is expected in file_names
for file_name in file_names_in_base_dir:
file_path = os.path.join(base_dir, file_name)
if not is_yaml_file(file_path):
logging.warning('Ignoring %s as the file does not end with .yaml or .yml',
file_path)
continue
with open(file_path, 'r') as f:
file_content = f.read()
if identify.get_operator_artifact_type(file_content) != 'Package':
logger.warning('Ignoring %s as it is not a valid package file.', file_name)
elif not packages:
packages.append(file_path)
else:
msg = f'The input source directory expects only 1 valid package file.'
logger.error(msg)
raise errors.OpCourierBadBundle(msg, {})
if not packages:
msg = f'The input source directory expects at least 1 valid package file.'
logger.error(msg)
raise errors.OpCourierBadBundle(msg, {})
return packages[0]
def get_crd_csv_files_info(
folder_path: str) -> Tuple[List[Tuple], List[Tuple]]:
"""
Given a folder path, the method returns the CRD and CSV files info parsed from the
input directory.
:param folder_path: the path of the input folder
:return: CRD and CSV files info parsed from the input directory. Each files_info
is a list of tuples, where each tuple contains two elements, namely
the file path and its content
"""
crd_files_info, csv_files_info = [], []
for item in os.listdir(folder_path):
item_path = os.path.join(folder_path, item)
if not os.path.isfile(item_path):
continue
if is_yaml_file(item_path):
with open(item_path) as f:
file_content = f.read()
file_type = identify.get_operator_artifact_type(file_content)
if file_type == CRD_STR:
crd_files_info.append((item_path, file_content))
elif file_type == CSV_STR:
csv_files_info.append((item_path, file_content))
return crd_files_info, csv_files_info
def get_folder_semver(folder_path: str):
for item in os.listdir(folder_path):
item_path = os.path.join(folder_path, item)
if not os.path.isfile(item_path) or not is_yaml_file(item_path):
continue
with open(item_path, 'r') as f:
file_content = f.read()
if identify.get_operator_artifact_type(
file_content) == 'ClusterServiceVersion':
try:
csv_version = safe_load(file_content)['spec']['version']
except MarkedYAMLError:
msg = f'{item} is not a valid YAML file.'
logger.error(msg)
raise OpCourierBadYaml(msg)
except KeyError:
msg = f'{item} is not a valid CSV file as "spec.version" ' \
f'field is required'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
return csv_version
return None
def _updateBundle(self, operatorBundle, file_name, yaml_string):
# Determine which operator file type the yaml is
operator_artifact = identify.get_operator_artifact_type(yaml_string)
# If the file isn't one of our special types, we ignore it and return
if operator_artifact == identify.UNKNOWN_FILE:
return operatorBundle
# Get the array name expected by the dictionary for the given file type
op_artifact_plural = operator_artifact[0:1].lower(
) + operator_artifact[1:] + 's'
# Marshal the yaml into a dictionary
yaml_data = yaml.safe_load(yaml_string)
# Add the data dictionary to the correct list
operatorBundle["data"][op_artifact_plural].append(yaml_data)
# Encode the dictionary into a string, then use that as a key to reference
# the file name associated with that yaml file. Then add it to the metadata.
if file_name != "":
unencoded_yaml = yaml.dump(yaml_data)
relative_path = self._get_relative_path(file_name)
operatorBundle["metadata"]["filenames"][hash(
unencoded_yaml)] = relative_path
return operatorBundle
def get_manifests_info(self, source_dir):
"""
Given a source directory OR a list of yaml files. The function returns a dict
containing all operator bundle file information grouped by version. Note that only
one of source_dir or yamls can be specified.
:param source_dir: Path to local directory of operator bundles, which can be
either flat or nested
:param yamls: A list of yaml strings to create bundle with
:return: A dictionary object where the key is the semantic version of each bundle,
and the value is a list of yaml strings of operator bundle files.
FLAT_KEY is used as key if the directory structure is flat
"""
# VERSION => manifest_files_content
# FLAT_KEY is used as key to indicate the flat directory structure
manifests = {}
root_path, dir_names, root_dir_files = next(os.walk(source_dir))
# removing dirs whose name are not semver
version_dirs = list(
filter(lambda x: self.is_dir_name_semver(x), dir_names))
# flat directory
if not version_dirs:
manifests[FLAT_KEY] = self.get_manifest_files_content(
[os.path.join(root_path, file) for file in root_dir_files])
# nested
else:
# add all manifest files from each version folder to manifests dict
for version_dir in version_dirs:
version_dir_path = os.path.join(root_path, version_dir)
_, _, version_dir_files = next(os.walk(version_dir_path))
file_paths = [
os.path.join(version_dir_path, file)
for file in version_dir_files
]
manifests[version_dir] = self.get_manifest_files_content(
file_paths)
# get the package file from root dir and add to each version of manifest
package_content = None
for root_dir_file in root_dir_files:
with open(os.path.join(root_path, root_dir_file), 'r') as f:
file_content = f.read()
if identify.get_operator_artifact_type(
file_content) == 'Package':
# ensure only 1 package is found in root directory
if package_content:
msg = 'There should be only 1 package file defined ' \
'in the source directory.'
logging.error(msg)
raise OpCourierBadBundle(msg, {})
package_content = file_content
if not package_content:
msg = 'No package file exists in the nested bundle.'
logging.error(msg)
raise OpCourierBadBundle(msg, {})
for version in manifests:
manifests[version].append(package_content)
return manifests
def get_csvs_pkg_info_from_root(source_dir: str) -> Tuple[List[Tuple], Tuple]:
"""
Given a source directory path, the method returns the CSVs and package file info
parsed from the input directory.
:param source_dir: the path of the input source folder
:return: CSVs and package file info parsed from the input directory.
csvs_info is a list of tuples whereas pkg_info is a single tuple, and
each tuple contains two elements, namely the file path and its content
"""
root_path, dir_names, root_dir_files = next(os.walk(source_dir))
root_file_paths = [
os.path.join(root_path, file) for file in root_dir_files
]
# [(CSV1_PATH, CSV1_CONTENT), ..., (CSVn_PATH, CSVn_CONTENT)]
csvs_info_list = []
# (PKG_PATH, PKG_CONTENT)
pkg_info = None
# check if package / csv is present in the source dir root, and
# populate the above two info variables
for root_file_path in root_file_paths:
if is_yaml_file(root_file_path):
with open(root_file_path) as f:
file_content = f.read()
file_type = identify.get_operator_artifact_type(file_content)
if file_type == CSV_STR:
csvs_info_list.append((root_file_path, file_content))
elif file_type == PKG_STR:
if pkg_info:
msg = 'Only 1 package is expected to exist in source root folder.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
pkg_info = (root_file_path, file_content)
if not pkg_info:
msg = 'Bundle does not contain any packages.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
return csvs_info_list, pkg_info
def is_manifest_folder(folder_path):
"""
:param folder_path: the path of the input folder
:return: True if the folder contains valid operator manifest files (at least
1 valid CSV file), False otherwise
"""
for item in os.listdir(folder_path):
item_path = os.path.join(folder_path, item)
if not os.path.isfile(item_path):
continue
if is_yaml_file(item_path):
with open(item_path) as f:
file_content = f.read()
if CSV_STR == identify.get_operator_artifact_type(file_content):
return True
folder_name = os.path.basename(folder_path)
logger.warning(
'Ignoring folder "%s" as it is not a valid manifest '
'folder', folder_name)
return False
def test_get_operator_artifact_type_assertions(fname):
with open(fname) as f:
yaml = f.read()
identify.get_operator_artifact_type(yaml)
def test_get_operator_artifact_type(fname, expected):
with open(fname) as f:
yaml = f.read()
assert identify.get_operator_artifact_type(yaml) == expected
def parse_manifest_folder(manifest_path: str, folder_semver: str,
csv_paths: list, crd_dict: Dict[str, Tuple[str,
str]]):
"""
Parse the version folder of the bundle and collect information of CSV and CRDs
in the bundle
:param manifest_path: The path of the manifest folder containing bundle files
:param folder_semver: The semantic version of the current folder
:param csv_paths: A list of CSV file paths inside version folders
:param crd_dict: dict that contains CRD info collected from different version folders,
where the key is the CRD name, and the value is a tuple where the first element is
the version of the bundle, and the second is the path of the CRD file
"""
logger.info('Parsing folder %s for operator version %s',
os.path.basename(manifest_path), folder_semver)
contains_csv = False
for item in os.listdir(manifest_path):
item_path = os.path.join(manifest_path, item)
if not os.path.isfile(item_path):
logger.warning('Ignoring %s as it is not a regular file.', item)
continue
if not is_yaml_file(item_path):
logger.warning(
'Ignoring %s as the file does not end with .yaml or .yml',
item_path)
continue
with open(item_path, 'r') as f:
file_content = f.read()
yaml_type = identify.get_operator_artifact_type(file_content)
if yaml_type == 'ClusterServiceVersion':
contains_csv = True
csv_paths.append(item_path)
elif yaml_type == 'CustomResourceDefinition':
try:
crd_name = safe_load(file_content)['metadata']['name']
except MarkedYAMLError:
msg = "Courier requires valid input YAML files"
logger.error(msg)
raise OpCourierBadYaml(msg)
except KeyError:
msg = f'{item} is not a valid CRD file as "metadata.name" ' \
f'field is required'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
# create new CRD type entry if not found in dict
if crd_name not in crd_dict:
crd_dict[crd_name] = (folder_semver, item_path)
# update the CRD type entry with the file with the newest version
elif semver.compare(folder_semver, crd_dict[crd_name][0]) > 0:
crd_dict[crd_name] = (crd_dict[crd_name][0], item_path)
if not contains_csv:
msg = 'This version directory does not contain any valid CSV file.'
logger.error(msg)
raise OpCourierBadBundle(msg, {})
def test_get_operator_artifact_type_assertions(fname):
with open(fname) as f:
yaml = f.read()
result = identify.get_operator_artifact_type(yaml)
assert result == identify.UNKNOWN_FILE
def parse_version_folder(base_dir: str, version_folder_name: str,
csv_paths: list, crd_dict: Dict[str, Tuple[str, str]]):
"""
Parse the version folder of the bundle and collect information of CSV and CRDs
in the bundle
:param base_dir: Path of the base directory where the version folder is located
:param version_folder_name: The name of the version folder containing bundle files
:param csv_paths: A list of CSV file paths inside version folders
:param crd_dict: dict that contains CRD info collected from different version folders,
where the key is the CRD name, and the value is a tuple where the first element is
the version of the bundle, and the second is the path of the CRD file
"""
# parse each version folder and parse CRD, CSV files
try:
semver.parse(version_folder_name)
except ValueError:
logger.warning("Ignoring %s as it is not a valid semver. "
"See https://semver.org for the semver specification.",
version_folder_name)
return
logger.info('Parsing folder: %s...', version_folder_name)
contains_csv = False
version_folder_path = os.path.join(base_dir, version_folder_name)
for item in os.listdir(os.path.join(base_dir, version_folder_name)):
item_path = os.path.join(version_folder_path, item)
if not os.path.isfile(item_path):
logger.warning('Ignoring %s as it is not a regular file.', item)
continue
if not is_yaml_file(item_path):
logging.warning('Ignoring %s as the file does not end with .yaml or .yml',
item_path)
continue
with open(item_path, 'r') as f:
file_content = f.read()
yaml_type = identify.get_operator_artifact_type(file_content)
if yaml_type == 'ClusterServiceVersion':
contains_csv = True
csv_paths.append(item_path)
elif yaml_type == 'CustomResourceDefinition':
try:
crd_name = safe_load(file_content)['metadata']['name']
except MarkedYAMLError:
msg = "Courier requires valid input YAML files"
logger.error(msg)
raise errors.OpCourierBadYaml(msg)
except KeyError:
msg = f'{item} is not a valid CRD file as "metadata.name" ' \
f'field is required'
logger.error(msg)
raise errors.OpCourierBadBundle(msg, {})
# create new CRD type entry if not found in dict
if crd_name not in crd_dict:
crd_dict[crd_name] = (version_folder_name, item_path)
# update the CRD type entry with the file with the newest version
elif semver.compare(version_folder_name, crd_dict[crd_name][0]) > 0:
crd_dict[crd_name] = (crd_dict[crd_name][0], item_path)
if not contains_csv:
msg = 'This version directory does not contain any valid CSV file.'
logger.error(msg)
raise errors.OpCourierBadBundle(msg, {})
def nest_flat_bundles(manifest_files_content, output_dir, temp_registry_dir):
package = {}
crds = {}
csvs = []
errors = []
# first lets parse all the files
for yaml_string in manifest_files_content:
yaml_type = identify.get_operator_artifact_type(yaml_string)
if yaml_type == PKG_STR:
if not package:
package = yaml.safe_load(yaml_string)
else:
errors.append("Multiple packages in directory.")
if yaml_type == CRD_STR:
crd = yaml.safe_load(yaml_string)
if "metadata" in crd and "name" in crd["metadata"]:
crd_name = crd["metadata"]["name"]
crds[crd_name] = crd
else:
errors.append("CRD has no `metadata.name` field defined")
if yaml_type == CSV_STR:
csv = yaml.safe_load(yaml_string)
csvs.append(csv)
if len(csvs) == 0:
errors.append("No csvs in directory.")
if not package:
errors.append("No package file in directory.")
# write the package file
if "packageName" in package:
package_name = package["packageName"]
packagefile_name = os.path.join(temp_registry_dir,
'%s.package.yaml' % package_name)
with open(packagefile_name, 'w') as outfile:
yaml.dump(package, outfile, default_flow_style=False)
outfile.flush()
# now lets create a subdirectory for each version of the csv,
# and add all the relevant crds to it
for csv in csvs:
if "metadata" not in csv:
errors.append("CSV has no `metadata` field defined")
continue
if "name" not in csv["metadata"]:
errors.append("CSV has no `metadata.name` field defined")
continue
csv_name = csv["metadata"]["name"]
if "spec" not in csv:
errors.append("CSV %s has no `spec` field defined" % csv_name)
continue
if "version" not in csv["spec"]:
errors.append("CSV %s has no `spec.version` field defined" %
csv_name)
continue
version = csv["spec"]["version"]
csv_folder = temp_registry_dir + "/" + version
if not os.path.exists(csv_folder):
os.makedirs(csv_folder)
csv_path = os.path.join(csv_folder,
f'{csv_name}.clusterserviceversion.yaml')
with open(csv_path, 'w') as outfile:
yaml.dump(csv, outfile, default_flow_style=False)
outfile.flush()
if "customresourcedefinitions" in csv["spec"]:
if "owned" in csv["spec"]["customresourcedefinitions"]:
csv_crds = csv["spec"]["customresourcedefinitions"][
"owned"]
for csv_crd in csv_crds:
if "name" not in csv_crd:
errors.append(
"CSV %s has an owned CRD without a `name`"
"field defined" % csv_name)
continue
crd_name = csv_crd["name"]
if crd_name in crds:
crd = crds[crd_name]
crdfile_name = os.path.join(
csv_folder, '%s.crd.yaml' % crd_name)
with open(crdfile_name, 'w') as outfile:
yaml.dump(crd,
outfile,
default_flow_style=False)
outfile.flush()
else:
errors.append(
"CRD %s mentioned in CSV %s was not found"
"in directory." % (crd_name, csv_name))
else:
errors.append("Package file has no `packageName` field defined")
# if no errors were encountered, lets create the real directory and populate it.
if len(errors) == 0:
if not os.path.exists(output_dir):
os.makedirs(output_dir)
copy_tree(temp_registry_dir, output_dir)
else:
for err in errors:
logger.error(err)
def _get_field_entry(self, yamlContent):
yaml_type = identify.get_operator_artifact_type(yamlContent)
return yaml_type[0:1].lower() + yaml_type[1:] + 's'
def nest_bundles(yaml_files, registry_dir, temp_registry_dir):
package = {}
crds = {}
csvs = []
errors = []
# first lets parse all the files
for yaml_string in yaml_files:
yaml_type = identify.get_operator_artifact_type(yaml_string)
if yaml_type == "Package":
if not package:
package = yaml.safe_load(yaml_string)
else:
errors.append("Multiple packages in directory.")
if yaml_type == "CustomResourceDefinition":
crd = yaml.safe_load(yaml_string)
crd_name = crd["metadata"]["name"]
crds[crd_name] = crd
if yaml_type == "ClusterServiceVersion":
csv = yaml.safe_load(yaml_string)
csvs.append(csv)
if len(csvs) == 0:
errors.append("No csvs in directory.")
if not package:
errors.append("No package file in directory.")
# write the package file
package_name = package["packageName"]
with open('%s/%s.package.yaml' % (temp_registry_dir, package_name),
'w') as outfile:
yaml.dump(package, outfile, default_flow_style=False)
outfile.flush()
# now lets create a subdirectory for each version of the csv,
# and add all the relevant crds to it
for csv in csvs:
csv_name = csv["metadata"]["name"]
version = csv["spec"]["version"]
csv_folder = temp_registry_dir + "/" + version
if not os.path.exists(csv_folder):
os.makedirs(csv_folder)
csv_path = '%s/%s.clusterserviceversion.yaml' % (csv_folder, csv_name)
with open(csv_path, 'w') as outfile:
yaml.dump(csv, outfile, default_flow_style=False)
outfile.flush()
csv_crds = csv["spec"]["customresourcedefinitions"]["owned"]
for csv_crd in csv_crds:
crd_name = csv_crd["name"]
if crd_name in crds:
crd = crds[crd_name]
with open('%s/%s.crd.yaml' % (csv_folder, crd_name),
'w') as outfile:
yaml.dump(crd, outfile, default_flow_style=False)
outfile.flush()
else:
errors.append(
"CRD %s mentioned in CSV %s was not found in directory." %
(crd_name, csv_name))
# if no errors were encountered, lets create the real directory and populate it.
if len(errors) == 0:
if not os.path.exists(registry_dir):
os.makedirs(registry_dir)
copy_tree(temp_registry_dir, registry_dir)
else:
for err in errors:
logger.error(err)
