def test_get_operator_artifact_type_assertions(fname): with open(fname) as f: yaml = f.read() with pytest.raises(OpCourierBadArtifact) as e, LogCapture() as logs: identify.get_operator_artifact_type(yaml) logs.check(('operatorcourier.identify', 'ERROR', 'Courier requires valid CSV, CRD, and Package files'), ) assert 'Courier requires valid CSV, CRD, and Package files' == str(e.value)
def test_get_operator_artifact_type_with_invalid_yaml(fname): with open(fname) as f: yaml = f.read() with pytest.raises(OpCourierBadYaml) as e, LogCapture() as logs: identify.get_operator_artifact_type(yaml) logs.check(('operatorcourier.identify', 'ERROR', 'Courier requires valid input YAML files'), ) assert 'Courier requires valid input YAML files' == str(e.value)
def get_package_path(base_dir: str, file_names_in_base_dir: list) -> str: packages = [] # add package file to file_paths_to_copy # only 1 package yaml file is expected in file_names for file_name in file_names_in_base_dir: file_path = os.path.join(base_dir, file_name) if not is_yaml_file(file_path): logging.warning('Ignoring %s as the file does not end with .yaml or .yml', file_path) continue with open(file_path, 'r') as f: file_content = f.read() if identify.get_operator_artifact_type(file_content) != 'Package': logger.warning('Ignoring %s as it is not a valid package file.', file_name) elif not packages: packages.append(file_path) else: msg = f'The input source directory expects only 1 valid package file.' logger.error(msg) raise errors.OpCourierBadBundle(msg, {}) if not packages: msg = f'The input source directory expects at least 1 valid package file.' logger.error(msg) raise errors.OpCourierBadBundle(msg, {}) return packages[0]
def get_crd_csv_files_info( folder_path: str) -> Tuple[List[Tuple], List[Tuple]]: """ Given a folder path, the method returns the CRD and CSV files info parsed from the input directory. :param folder_path: the path of the input folder :return: CRD and CSV files info parsed from the input directory. Each files_info is a list of tuples, where each tuple contains two elements, namely the file path and its content """ crd_files_info, csv_files_info = [], [] for item in os.listdir(folder_path): item_path = os.path.join(folder_path, item) if not os.path.isfile(item_path): continue if is_yaml_file(item_path): with open(item_path) as f: file_content = f.read() file_type = identify.get_operator_artifact_type(file_content) if file_type == CRD_STR: crd_files_info.append((item_path, file_content)) elif file_type == CSV_STR: csv_files_info.append((item_path, file_content)) return crd_files_info, csv_files_info
def get_folder_semver(folder_path: str): for item in os.listdir(folder_path): item_path = os.path.join(folder_path, item) if not os.path.isfile(item_path) or not is_yaml_file(item_path): continue with open(item_path, 'r') as f: file_content = f.read() if identify.get_operator_artifact_type( file_content) == 'ClusterServiceVersion': try: csv_version = safe_load(file_content)['spec']['version'] except MarkedYAMLError: msg = f'{item} is not a valid YAML file.' logger.error(msg) raise OpCourierBadYaml(msg) except KeyError: msg = f'{item} is not a valid CSV file as "spec.version" ' \ f'field is required' logger.error(msg) raise OpCourierBadBundle(msg, {}) return csv_version return None
def _updateBundle(self, operatorBundle, file_name, yaml_string): # Determine which operator file type the yaml is operator_artifact = identify.get_operator_artifact_type(yaml_string) # If the file isn't one of our special types, we ignore it and return if operator_artifact == identify.UNKNOWN_FILE: return operatorBundle # Get the array name expected by the dictionary for the given file type op_artifact_plural = operator_artifact[0:1].lower( ) + operator_artifact[1:] + 's' # Marshal the yaml into a dictionary yaml_data = yaml.safe_load(yaml_string) # Add the data dictionary to the correct list operatorBundle["data"][op_artifact_plural].append(yaml_data) # Encode the dictionary into a string, then use that as a key to reference # the file name associated with that yaml file. Then add it to the metadata. if file_name != "": unencoded_yaml = yaml.dump(yaml_data) relative_path = self._get_relative_path(file_name) operatorBundle["metadata"]["filenames"][hash( unencoded_yaml)] = relative_path return operatorBundle
def get_manifests_info(self, source_dir): """ Given a source directory OR a list of yaml files. The function returns a dict containing all operator bundle file information grouped by version. Note that only one of source_dir or yamls can be specified. :param source_dir: Path to local directory of operator bundles, which can be either flat or nested :param yamls: A list of yaml strings to create bundle with :return: A dictionary object where the key is the semantic version of each bundle, and the value is a list of yaml strings of operator bundle files. FLAT_KEY is used as key if the directory structure is flat """ # VERSION => manifest_files_content # FLAT_KEY is used as key to indicate the flat directory structure manifests = {} root_path, dir_names, root_dir_files = next(os.walk(source_dir)) # removing dirs whose name are not semver version_dirs = list( filter(lambda x: self.is_dir_name_semver(x), dir_names)) # flat directory if not version_dirs: manifests[FLAT_KEY] = self.get_manifest_files_content( [os.path.join(root_path, file) for file in root_dir_files]) # nested else: # add all manifest files from each version folder to manifests dict for version_dir in version_dirs: version_dir_path = os.path.join(root_path, version_dir) _, _, version_dir_files = next(os.walk(version_dir_path)) file_paths = [ os.path.join(version_dir_path, file) for file in version_dir_files ] manifests[version_dir] = self.get_manifest_files_content( file_paths) # get the package file from root dir and add to each version of manifest package_content = None for root_dir_file in root_dir_files: with open(os.path.join(root_path, root_dir_file), 'r') as f: file_content = f.read() if identify.get_operator_artifact_type( file_content) == 'Package': # ensure only 1 package is found in root directory if package_content: msg = 'There should be only 1 package file defined ' \ 'in the source directory.' logging.error(msg) raise OpCourierBadBundle(msg, {}) package_content = file_content if not package_content: msg = 'No package file exists in the nested bundle.' logging.error(msg) raise OpCourierBadBundle(msg, {}) for version in manifests: manifests[version].append(package_content) return manifests
def get_csvs_pkg_info_from_root(source_dir: str) -> Tuple[List[Tuple], Tuple]: """ Given a source directory path, the method returns the CSVs and package file info parsed from the input directory. :param source_dir: the path of the input source folder :return: CSVs and package file info parsed from the input directory. csvs_info is a list of tuples whereas pkg_info is a single tuple, and each tuple contains two elements, namely the file path and its content """ root_path, dir_names, root_dir_files = next(os.walk(source_dir)) root_file_paths = [ os.path.join(root_path, file) for file in root_dir_files ] # [(CSV1_PATH, CSV1_CONTENT), ..., (CSVn_PATH, CSVn_CONTENT)] csvs_info_list = [] # (PKG_PATH, PKG_CONTENT) pkg_info = None # check if package / csv is present in the source dir root, and # populate the above two info variables for root_file_path in root_file_paths: if is_yaml_file(root_file_path): with open(root_file_path) as f: file_content = f.read() file_type = identify.get_operator_artifact_type(file_content) if file_type == CSV_STR: csvs_info_list.append((root_file_path, file_content)) elif file_type == PKG_STR: if pkg_info: msg = 'Only 1 package is expected to exist in source root folder.' logger.error(msg) raise OpCourierBadBundle(msg, {}) pkg_info = (root_file_path, file_content) if not pkg_info: msg = 'Bundle does not contain any packages.' logger.error(msg) raise OpCourierBadBundle(msg, {}) return csvs_info_list, pkg_info
def is_manifest_folder(folder_path): """ :param folder_path: the path of the input folder :return: True if the folder contains valid operator manifest files (at least 1 valid CSV file), False otherwise """ for item in os.listdir(folder_path): item_path = os.path.join(folder_path, item) if not os.path.isfile(item_path): continue if is_yaml_file(item_path): with open(item_path) as f: file_content = f.read() if CSV_STR == identify.get_operator_artifact_type(file_content): return True folder_name = os.path.basename(folder_path) logger.warning( 'Ignoring folder "%s" as it is not a valid manifest ' 'folder', folder_name) return False
def test_get_operator_artifact_type_assertions(fname): with open(fname) as f: yaml = f.read() identify.get_operator_artifact_type(yaml)
def test_get_operator_artifact_type(fname, expected): with open(fname) as f: yaml = f.read() assert identify.get_operator_artifact_type(yaml) == expected
def parse_manifest_folder(manifest_path: str, folder_semver: str, csv_paths: list, crd_dict: Dict[str, Tuple[str, str]]): """ Parse the version folder of the bundle and collect information of CSV and CRDs in the bundle :param manifest_path: The path of the manifest folder containing bundle files :param folder_semver: The semantic version of the current folder :param csv_paths: A list of CSV file paths inside version folders :param crd_dict: dict that contains CRD info collected from different version folders, where the key is the CRD name, and the value is a tuple where the first element is the version of the bundle, and the second is the path of the CRD file """ logger.info('Parsing folder %s for operator version %s', os.path.basename(manifest_path), folder_semver) contains_csv = False for item in os.listdir(manifest_path): item_path = os.path.join(manifest_path, item) if not os.path.isfile(item_path): logger.warning('Ignoring %s as it is not a regular file.', item) continue if not is_yaml_file(item_path): logger.warning( 'Ignoring %s as the file does not end with .yaml or .yml', item_path) continue with open(item_path, 'r') as f: file_content = f.read() yaml_type = identify.get_operator_artifact_type(file_content) if yaml_type == 'ClusterServiceVersion': contains_csv = True csv_paths.append(item_path) elif yaml_type == 'CustomResourceDefinition': try: crd_name = safe_load(file_content)['metadata']['name'] except MarkedYAMLError: msg = "Courier requires valid input YAML files" logger.error(msg) raise OpCourierBadYaml(msg) except KeyError: msg = f'{item} is not a valid CRD file as "metadata.name" ' \ f'field is required' logger.error(msg) raise OpCourierBadBundle(msg, {}) # create new CRD type entry if not found in dict if crd_name not in crd_dict: crd_dict[crd_name] = (folder_semver, item_path) # update the CRD type entry with the file with the newest version elif semver.compare(folder_semver, crd_dict[crd_name][0]) > 0: crd_dict[crd_name] = (crd_dict[crd_name][0], item_path) if not contains_csv: msg = 'This version directory does not contain any valid CSV file.' logger.error(msg) raise OpCourierBadBundle(msg, {})
def test_get_operator_artifact_type_assertions(fname): with open(fname) as f: yaml = f.read() result = identify.get_operator_artifact_type(yaml) assert result == identify.UNKNOWN_FILE
def parse_version_folder(base_dir: str, version_folder_name: str, csv_paths: list, crd_dict: Dict[str, Tuple[str, str]]): """ Parse the version folder of the bundle and collect information of CSV and CRDs in the bundle :param base_dir: Path of the base directory where the version folder is located :param version_folder_name: The name of the version folder containing bundle files :param csv_paths: A list of CSV file paths inside version folders :param crd_dict: dict that contains CRD info collected from different version folders, where the key is the CRD name, and the value is a tuple where the first element is the version of the bundle, and the second is the path of the CRD file """ # parse each version folder and parse CRD, CSV files try: semver.parse(version_folder_name) except ValueError: logger.warning("Ignoring %s as it is not a valid semver. " "See https://semver.org for the semver specification.", version_folder_name) return logger.info('Parsing folder: %s...', version_folder_name) contains_csv = False version_folder_path = os.path.join(base_dir, version_folder_name) for item in os.listdir(os.path.join(base_dir, version_folder_name)): item_path = os.path.join(version_folder_path, item) if not os.path.isfile(item_path): logger.warning('Ignoring %s as it is not a regular file.', item) continue if not is_yaml_file(item_path): logging.warning('Ignoring %s as the file does not end with .yaml or .yml', item_path) continue with open(item_path, 'r') as f: file_content = f.read() yaml_type = identify.get_operator_artifact_type(file_content) if yaml_type == 'ClusterServiceVersion': contains_csv = True csv_paths.append(item_path) elif yaml_type == 'CustomResourceDefinition': try: crd_name = safe_load(file_content)['metadata']['name'] except MarkedYAMLError: msg = "Courier requires valid input YAML files" logger.error(msg) raise errors.OpCourierBadYaml(msg) except KeyError: msg = f'{item} is not a valid CRD file as "metadata.name" ' \ f'field is required' logger.error(msg) raise errors.OpCourierBadBundle(msg, {}) # create new CRD type entry if not found in dict if crd_name not in crd_dict: crd_dict[crd_name] = (version_folder_name, item_path) # update the CRD type entry with the file with the newest version elif semver.compare(version_folder_name, crd_dict[crd_name][0]) > 0: crd_dict[crd_name] = (crd_dict[crd_name][0], item_path) if not contains_csv: msg = 'This version directory does not contain any valid CSV file.' logger.error(msg) raise errors.OpCourierBadBundle(msg, {})
def nest_flat_bundles(manifest_files_content, output_dir, temp_registry_dir): package = {} crds = {} csvs = [] errors = [] # first lets parse all the files for yaml_string in manifest_files_content: yaml_type = identify.get_operator_artifact_type(yaml_string) if yaml_type == PKG_STR: if not package: package = yaml.safe_load(yaml_string) else: errors.append("Multiple packages in directory.") if yaml_type == CRD_STR: crd = yaml.safe_load(yaml_string) if "metadata" in crd and "name" in crd["metadata"]: crd_name = crd["metadata"]["name"] crds[crd_name] = crd else: errors.append("CRD has no `metadata.name` field defined") if yaml_type == CSV_STR: csv = yaml.safe_load(yaml_string) csvs.append(csv) if len(csvs) == 0: errors.append("No csvs in directory.") if not package: errors.append("No package file in directory.") # write the package file if "packageName" in package: package_name = package["packageName"] packagefile_name = os.path.join(temp_registry_dir, '%s.package.yaml' % package_name) with open(packagefile_name, 'w') as outfile: yaml.dump(package, outfile, default_flow_style=False) outfile.flush() # now lets create a subdirectory for each version of the csv, # and add all the relevant crds to it for csv in csvs: if "metadata" not in csv: errors.append("CSV has no `metadata` field defined") continue if "name" not in csv["metadata"]: errors.append("CSV has no `metadata.name` field defined") continue csv_name = csv["metadata"]["name"] if "spec" not in csv: errors.append("CSV %s has no `spec` field defined" % csv_name) continue if "version" not in csv["spec"]: errors.append("CSV %s has no `spec.version` field defined" % csv_name) continue version = csv["spec"]["version"] csv_folder = temp_registry_dir + "/" + version if not os.path.exists(csv_folder): os.makedirs(csv_folder) csv_path = os.path.join(csv_folder, f'{csv_name}.clusterserviceversion.yaml') with open(csv_path, 'w') as outfile: yaml.dump(csv, outfile, default_flow_style=False) outfile.flush() if "customresourcedefinitions" in csv["spec"]: if "owned" in csv["spec"]["customresourcedefinitions"]: csv_crds = csv["spec"]["customresourcedefinitions"][ "owned"] for csv_crd in csv_crds: if "name" not in csv_crd: errors.append( "CSV %s has an owned CRD without a `name`" "field defined" % csv_name) continue crd_name = csv_crd["name"] if crd_name in crds: crd = crds[crd_name] crdfile_name = os.path.join( csv_folder, '%s.crd.yaml' % crd_name) with open(crdfile_name, 'w') as outfile: yaml.dump(crd, outfile, default_flow_style=False) outfile.flush() else: errors.append( "CRD %s mentioned in CSV %s was not found" "in directory." % (crd_name, csv_name)) else: errors.append("Package file has no `packageName` field defined") # if no errors were encountered, lets create the real directory and populate it. if len(errors) == 0: if not os.path.exists(output_dir): os.makedirs(output_dir) copy_tree(temp_registry_dir, output_dir) else: for err in errors: logger.error(err)
def _get_field_entry(self, yamlContent): yaml_type = identify.get_operator_artifact_type(yamlContent) return yaml_type[0:1].lower() + yaml_type[1:] + 's'
def nest_bundles(yaml_files, registry_dir, temp_registry_dir): package = {} crds = {} csvs = [] errors = [] # first lets parse all the files for yaml_string in yaml_files: yaml_type = identify.get_operator_artifact_type(yaml_string) if yaml_type == "Package": if not package: package = yaml.safe_load(yaml_string) else: errors.append("Multiple packages in directory.") if yaml_type == "CustomResourceDefinition": crd = yaml.safe_load(yaml_string) crd_name = crd["metadata"]["name"] crds[crd_name] = crd if yaml_type == "ClusterServiceVersion": csv = yaml.safe_load(yaml_string) csvs.append(csv) if len(csvs) == 0: errors.append("No csvs in directory.") if not package: errors.append("No package file in directory.") # write the package file package_name = package["packageName"] with open('%s/%s.package.yaml' % (temp_registry_dir, package_name), 'w') as outfile: yaml.dump(package, outfile, default_flow_style=False) outfile.flush() # now lets create a subdirectory for each version of the csv, # and add all the relevant crds to it for csv in csvs: csv_name = csv["metadata"]["name"] version = csv["spec"]["version"] csv_folder = temp_registry_dir + "/" + version if not os.path.exists(csv_folder): os.makedirs(csv_folder) csv_path = '%s/%s.clusterserviceversion.yaml' % (csv_folder, csv_name) with open(csv_path, 'w') as outfile: yaml.dump(csv, outfile, default_flow_style=False) outfile.flush() csv_crds = csv["spec"]["customresourcedefinitions"]["owned"] for csv_crd in csv_crds: crd_name = csv_crd["name"] if crd_name in crds: crd = crds[crd_name] with open('%s/%s.crd.yaml' % (csv_folder, crd_name), 'w') as outfile: yaml.dump(crd, outfile, default_flow_style=False) outfile.flush() else: errors.append( "CRD %s mentioned in CSV %s was not found in directory." % (crd_name, csv_name)) # if no errors were encountered, lets create the real directory and populate it. if len(errors) == 0: if not os.path.exists(registry_dir): os.makedirs(registry_dir) copy_tree(temp_registry_dir, registry_dir) else: for err in errors: logger.error(err)