def validate_driver_types_missing_selectors(self, journal, validation_options, errors, warnings, descriptor_path): if os.path.exists(descriptor_path): descriptor = descriptor_utils.DescriptorParser().read_from_file(descriptor_path) driver_entries_needing_fix = self.__gather_driver_types_missing_selector_key(journal, validation_options, errors, warnings, descriptor, 'lifecycle') driver_entries_needing_fix.extend(self.__gather_driver_types_missing_selector_key(journal, validation_options, errors, warnings, descriptor, 'operations')) if 'default-driver' in descriptor.raw and isinstance(descriptor.default_driver, dict): for driver_type, driver_entry in descriptor.default_driver.items(): if 'infrastructure-type' in driver_entry: driver_entries_needing_fix.append(driver_entry) if len(driver_entries_needing_fix) > 0: if validation_options.allow_autocorrect == True: journal.event('Found lifecycle/operation/default-driver entries missing \'selector\' key before \'infrastructure-type\' [{0}], attempting to autocorrect by moving contents under a selector key'.format(descriptor_path)) managed_to_autocorrect = False autocorrect_error = None try: for entry_needing_fix in driver_entries_needing_fix: entry_needing_fix['selector'] = {'infrastructure-type': entry_needing_fix['infrastructure-type']} entry_needing_fix.pop('infrastructure-type') descriptor_utils.DescriptorParser().write_to_file(descriptor, descriptor_path) managed_to_autocorrect = True except Exception as e: autocorrect_error = e if not managed_to_autocorrect: msg = 'Found lifecycle/operation/default-driver entries missing \'selector\' key before \'infrastructure-type\' [{0}]: Unable to autocorrect this issue, please add this information to the Resource descriptor manually instead'.format(descriptor_path) if autocorrect_error is not None: msg += ' (autocorrect error={0})'.format(str(autocorrect_error)) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return else: msg = 'Found lifecycle/operation/default-driver entries missing \'selector\' key before \'infrastructure-type\' [{0}]: this format is no longer supported by the Brent Resource Manager. Move infrastructure-type information under the selector key or enable the autocorrect option'.format(descriptor_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return
def validate_unsupported_infrastructure(self, journal, validation_options, errors, warnings, descriptor_path, inf_path, lifecycle_path): if os.path.exists(descriptor_path): descriptor = descriptor_utils.DescriptorParser().read_from_file(descriptor_path) if 'infrastructure' in descriptor.raw and isinstance(descriptor.infrastructure, dict): entry_needing_lifecycle_fix = [] entry_needing_discover_fix = [] for inf_type, inf_entry in descriptor.infrastructure.items(): if 'template' in inf_entry: entry_needing_lifecycle_fix.append(inf_type) if 'discover' in inf_entry: entry_needing_discover_fix.append(inf_type) if (len(entry_needing_lifecycle_fix) + len(entry_needing_discover_fix)) > 0: if validation_options.allow_autocorrect == True: journal.event('Found unsupported infrastructure entries referencing templates [{0}], attempting to autocorrect by moving contents to Create/Delete/queries entries in descriptor'.format(descriptor_path)) managed_to_autocorrect = False autocorrect_error = None try: entry_needing_fix = entry_needing_lifecycle_fix + entry_needing_discover_fix for inf_type in entry_needing_fix: driver_type = inf_type if inf_type == 'Openstack': driver_type = 'openstack' elif inf_type == 'Kubernetes': driver_type = 'kubernetes' target_lifecycle_path = os.path.join(lifecycle_path, driver_type) if not os.path.exists(target_lifecycle_path): os.makedirs(target_lifecycle_path) inf_entry = descriptor.infrastructure[inf_type] self.__move_template_files(inf_path, target_lifecycle_path, inf_type, inf_entry) if inf_type in entry_needing_lifecycle_fix: create_properties = None if inf_type == 'Openstack' and inf_entry.get('template', {}).get('template-type') == 'TOSCA': create_properties = { 'template-type': { 'value': inf_entry.get('template', {}).get('template-type') } } self.__add_driver_to_lifecycle(descriptor, driver_type, inf_type, create_properties=create_properties) if inf_type in entry_needing_discover_fix: self.__add_driver_to_queries(descriptor, driver_type, inf_type) descriptor.infrastructure[inf_type] = {} descriptor_utils.DescriptorParser().write_to_file(descriptor, descriptor_path) self.rename_inf_path = True managed_to_autocorrect = True except Exception as e: autocorrect_error = e raise if not managed_to_autocorrect: msg = 'Found unsupported infrastructure entries referencing templates [{0}]: this format is no longer supported by the Brent Resource Manager. Unable to autocorrect this issue, please add this information to the Create/Delete lifecycle and/or queries manually instead'.format(descriptor_path) if autocorrect_error is not None: msg += ' (autocorrect error={0})'.format(str(autocorrect_error)) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return else: msg = 'Found infrastructure entries referencing templates [{0}]: this format is no longer supported by the Brent Resource Manager. Add this information to the Create/Delete lifecycle and/or queries instead or enable the autocorrect option'.format(descriptor_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return
def __validate_csar(self, journal, errors, warnings): journal.stage('Checking CSAR exists for {0}'.format(self.meta.name)) csar_path = self.tree.gen_csar_file_path(self.meta.full_name) if not os.path.exists(csar_path): msg = 'No CSAR found at: {0}'.format(csar_path) journal.error_event(msg) errors.append(validation.ValidationViolation(msg))
def __validate_descriptor(self, journal, errors, warnings): descriptor_path = self.tree.gen_root_descriptor_file_path( self.meta.full_name) if not os.path.exists(descriptor_path): msg = 'No descriptor found at: {0}'.format(descriptor_path) journal.error_event(msg) errors.append(validation.ValidationViolation(msg))
def __validate_res_pkg(self, journal, errors, warnings): journal.stage('Checking Resource package exists for {0}'.format(self.meta.name)) res_pkg_path = self.tree.gen_resource_package_file_path(self.meta.full_name) if not os.path.exists(res_pkg_path): msg = 'No Resource package found at: {0}'.format(res_pkg_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg))
def validate_descriptor(self, descriptor_path, errors, warnings, allow_autocorrect=False, is_template=False): if not os.path.exists(descriptor_path): msg = 'No descriptor found at: {0}'.format(descriptor_path) self.journal.error_event(msg) errors.append(validation.ValidationViolation(msg)) return self.journal.event('Checking descriptor found at: {0}'.format(descriptor_path)) descriptor = None try: descriptor = descriptor_utils.DescriptorParser().read_from_file(descriptor_path) except descriptor_utils.DescriptorParsingError as e: errors.append(validation.ValidationViolation('Descriptor [{0}]: could not be parsed: {1}'.format(descriptor_path, str(e)))) else: if descriptor.has_name() is True: type_invalid = False name_invalid = False version_invalid = False descriptor_type, descriptor_name, descriptor_version = descriptor.get_split_name() expected_type = descriptor_utils.ASSEMBLY_DESCRIPTOR_TYPE if is_template: expected_type = descriptor_utils.ASSEMBLY_TEMPLATE_DESCRIPTOR_TYPE if self.source_config.is_resource_project(): expected_type = descriptor_utils.RESOURCE_DESCRIPTOR_TYPE elif self.source_config.is_type_project(): expected_type = descriptor_utils.TYPE_DESCRIPTOR_TYPE if descriptor_type != expected_type: errors.append(validation.ValidationViolation('Descriptor [{0}]: name \'{1}\' includes type \'{2}\' but this should be \'{3}\' based on project configuration'.format(descriptor_path, descriptor.get_name(), descriptor_type, expected_type))) if descriptor_name != self.source_config.full_name: errors.append(validation.ValidationViolation('Descriptor [{0}]: name \'{1}\' includes \'{2}\' but this should be \'{3}\' based on project configuration'.format(descriptor_path, descriptor.get_name(), descriptor_name, self.source_config.full_name))) if descriptor_version != self.source_config.version: errors.append(validation.ValidationViolation('Descriptor [{0}]: name \'{1}\' includes version \'{2}\' but this should be \'{3}\' based on project configuration'.format(descriptor_path, descriptor.get_name(), descriptor_version, self.source_config.version))) if type_invalid or name_invalid or version_invalid: self.journal.error_event('Descriptor name validation failed') if not isinstance(descriptor.lifecycle, dict) and allow_autocorrect is True: self.journal.event('Found lifecycle list structure in Resource descriptor [{0}], attempting to autocorrect to latest structure'.format(descriptor_path)) new_lifecycle = {} for lifecycle in descriptor.lifecycle: new_lifecycle[lifecycle] = {} descriptor.lifecycle = new_lifecycle try: descriptor_utils.DescriptorParser().write_to_file(descriptor, descriptor_path) except Exception as e: self.journal.error_event('Failed to update lifecycle list structure in Resource descriptor [{0}]: {1}'.format(descriptor_path, str(e)))
def __validate_meta_inf(self, journal, errors, warnings): meta_inf_path = self.tree.meta_inf_manifest_file_path if not os.path.exists(meta_inf_path): msg = 'No manifest found at: {0}'.format(meta_inf_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) else: journal.event('Manifest found at: {0}'.format(meta_inf_path))
def __validate_lifecycle(self, journal, errors, warnings): lifecycle_path = self.tree.lifecycle_path if not os.path.exists(lifecycle_path): msg = 'No lifecycle sources found at: {0}'.format(lifecycle_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) else: journal.event('Lifecycle found at: {0}'.format(lifecycle_path))
def __rename_infrastructure_dir(self, journal, validation_options, errors, warnings, inf_path): #Rename infrastructure directory to keep as a backup, rather than remove it if os.path.exists(inf_path): try: os.rename(inf_path, inf_path + '-bak') except Exception as e: msg = 'Failed to rename infrastructure directory {0} to {1}, this directory is no longer needed after correcting infrastructure validation errors, please remove. Rename error was: {2}'.format(inf_path, inf_path + '-bak', str(e)) journal.error_event(msg) warnings.append(project_validation.ValidationViolation(msg))
def __find_or_error(self, journal, errors, warnings, path, artifact_type): if not os.path.exists(path): msg = 'No {0} found at: {1}'.format(artifact_type, path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return False else: journal.event('{0} found at: {1}'.format(artifact_type, path)) return True
def __validate_unsupported_infrastructure_manifest(self, journal, validation_options, errors, warnings): inf_manifest_path = self.tree.infrastructure_manifest_file_path if os.path.exists(inf_manifest_path): if validation_options.allow_autocorrect == True: journal.event('Found unsupported infrastructure manifest [{0}], attempting to autocorrect by moving contents to Resource descriptor'.format(inf_manifest_path)) managed_to_autocorrect = False autocorrect_error = None try: with open(inf_manifest_path, 'r') as f: inf_manifest_content = yaml.safe_load(f.read()) descriptor = descriptor_utils.DescriptorParser().read_from_file(self.get_main_descriptor()) descriptor.is_2_dot_1 = True if 'templates' in inf_manifest_content: for template_entry in inf_manifest_content['templates']: if 'infrastructure_type' in template_entry: infrastructure_type = template_entry['infrastructure_type'] template_file = template_entry.get('file', None) template_type = template_entry.get('template_type', None) descriptor.insert_infrastructure_template(infrastructure_type, template_file, template_type=template_type) if 'discover' in inf_manifest_content: for discover_entry in inf_manifest_content['discover']: if 'infrastructure_type' in discover_entry: infrastructure_type = discover_entry['infrastructure_type'] template_file = discover_entry.get('file', None) template_type = discover_entry.get('template_type', None) descriptor.insert_infrastructure_discover(infrastructure_type, template_file, template_type=template_type) descriptor_utils.DescriptorParser().write_to_file(descriptor, self.get_main_descriptor()) os.rename(inf_manifest_path, inf_manifest_path + '.bak') managed_to_autocorrect = True except Exception as e: autocorrect_error = e if not managed_to_autocorrect: msg = 'Found infrastructure manifest [{0}]: this file is no longer supported by the Brent Resource Manager. Unable to autocorrect this issue, please add this information to the Resource descriptor manually instead'.format(inf_manifest_path) if autocorrect_error is not None: msg += ' (autocorrect error={0})'.format(str(autocorrect_error)) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return else: msg = 'Found infrastructure manifest [{0}]: this file is no longer supported by the Brent Resource Manager. Add this information to the Resource descriptor instead or enable the autocorrect option'.format(inf_manifest_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return
def validate_unsupported_lifecycle_manifest(self, journal, validation_options, errors, warnings, lifecycle_manifest_path, descriptor_path): if os.path.exists(lifecycle_manifest_path) and os.path.exists(descriptor_path): if validation_options.allow_autocorrect == True: journal.event('Found unsupported lifecycle manifest [{0}], attempting to autocorrect by moving contents to Resource descriptor'.format(lifecycle_manifest_path)) managed_to_autocorrect = False autocorrect_error = None try: with open(lifecycle_manifest_path, 'r') as f: lifecycle_manifest_content = yaml.safe_load(f.read()) descriptor = descriptor_utils.DescriptorParser().read_from_file(descriptor_path) if 'types' in lifecycle_manifest_content: for entry in lifecycle_manifest_content['types']: if 'lifecycle_type' in entry and 'infrastructure_type' in entry: lifecycle_type = entry['lifecycle_type'] infrastructure_type = entry['infrastructure_type'] if lifecycle_type in descriptor.default_driver: if 'selector' not in descriptor.default_driver[lifecycle_type]: descriptor.default_driver[lifecycle_type]['selector'] = {} if 'infrastructure-type' not in descriptor.default_driver[lifecycle_type]['selector']: descriptor.default_driver[lifecycle_type]['selector']['infrastructure-type'] = [] descriptor.default_driver[lifecycle_type]['selector']['infrastructure-type'].append(infrastructure_type) else: descriptor.insert_default_driver(lifecycle_type, [infrastructure_type]) descriptor_utils.DescriptorParser().write_to_file(descriptor, descriptor_path) os.rename(lifecycle_manifest_path, lifecycle_manifest_path + '.bak') managed_to_autocorrect = True except Exception as e: autocorrect_error = e if not managed_to_autocorrect: msg = 'Found lifecycle manifest [{0}]: this file is no longer supported by the Brent Resource Manager. Unable to autocorrect this issue, please add this information to the Resource descriptor manually instead'.format(lifecycle_manifest_path) if autocorrect_error is not None: msg += ' (autocorrect error={0})'.format(str(autocorrect_error)) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return else: msg = 'Found lifecycle manifest [{0}]: this file is no longer supported by the Brent Resource Manager. Add this information to the Resource descriptor instead or enable the autocorrect option'.format(lifecycle_manifest_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) return
def __validate_res_pkg(self, journal, validation_options, errors, warnings): journal.stage('Checking Resource package exists for {0}'.format( self.meta.name)) res_pkg_path = self.tree.gen_resource_package_file_path( self.meta.full_name) if not os.path.exists(res_pkg_path): msg = 'No Resource package found at: {0}'.format(res_pkg_path) journal.error_event(msg) errors.append(project_validation.ValidationViolation(msg)) else: extraction_path = os.path.join(os.path.dirname(res_pkg_path), 'tmp-extract') os.makedirs(extraction_path) with zipfile.ZipFile(res_pkg_path, "r") as res_pkg: res_pkg.extractall(extraction_path) try: tree = BrentResourcePackageContentTree(extraction_path) BrentCorrectableValidation().validate_and_autocorrect(journal, validation_options, errors, warnings, tree.descriptor_file_path, \ tree.infrastructure_definitions_path, tree.infrastructure_manifest_file_path, tree.lifecycle_path, \ tree.lifecycle_manifest_file_path) with zipfile.ZipFile(res_pkg_path, "w") as res_pkg: res_pkg_content_tree = BrentResourcePackageContentTree() included_items = [{ 'path': tree.definitions_path, 'alias': res_pkg_content_tree.definitions_path }, { 'path': tree.lifecycle_path, 'alias': res_pkg_content_tree.lifecycle_path }] for included_item in included_items: self.__add_directory(journal, res_pkg, included_item) finally: if os.path.exists(extraction_path): shutil.rmtree(extraction_path)