def initiate_file_validator(self, validator_type): """ Run schema validate and file validate of file Returns: int 0 in case of success int 1 in case of error int 2 in case of skip """ if self.no_validate: print_color( f'Validator Skipped on file: {self.output_file} , no-validate flag was set.', LOG_COLORS.YELLOW) return SKIP_RETURN_CODE else: print_color('Starting validating files structure', LOG_COLORS.GREEN) if self.relative_content_path: structure_validator = StructureValidator( self.relative_content_path) validator = validator_type(structure_validator) if structure_validator.is_valid_file( ) and validator.is_valid_file(validate_rn=False): print_color('The files are valid', LOG_COLORS.GREEN) return SUCCESS_RETURN_CODE else: print_color('The files are invalid', LOG_COLORS.RED) return ERROR_RETURN_CODE else: print_color( f'The file {self.output_file} are not part of content repo, Validator Skipped', LOG_COLORS.YELLOW) return SKIP_RETURN_CODE
def test_is_file_valid(self, source, target, answer): try: copyfile(source, target) structure = StructureValidator(target) assert structure.is_valid_file() is answer finally: os.remove(target)
def test_is_file_valid(self, source, target, answer, mocker): mocker.patch.object(BaseValidator, 'check_file_flags', return_value='') try: copyfile(source, target) structure = StructureValidator(target) assert structure.is_valid_file() is answer finally: os.remove(target)
def initiate_file_validator(self, validator_type, scheme_type): print_color('Starting validating files structure', LOG_COLORS.GREEN) structure = StructureValidator(file_path=str(self.output_file_name), predefined_scheme=scheme_type) validator = validator_type(structure) if structure.is_valid_file() and validator.is_valid_file( validate_rn=False): print_color('The files are valid', LOG_COLORS.GREEN) return 0 else: print_color('The files are invalid', LOG_COLORS.RED) return 1
def initiate_file_validator(self, validator_type): """ Run schema validate and file validate of file Returns: int 0 in case of success int 1 in case of error int 2 in case of skip """ if self.no_validate: if self.verbose: click.secho( f'Validator Skipped on file: {self.output_file} , no-validate flag was set.', fg='yellow') return SKIP_RETURN_CODE else: if self.verbose: print_color('Starting validating files structure', LOG_COLORS.GREEN) # validates only on files in content repo if self.relative_content_path: file_type = find_type(self.output_file) # validates on the output file generated from the format structure_validator = StructureValidator( self.output_file, predefined_scheme=file_type, suppress_print=not self.verbose) validator = validator_type(structure_validator, suppress_print=not self.verbose) # TODO: remove the connection condition if we implement a specific validator for connections. if structure_validator.is_valid_file() and \ (file_type == FileType.CONNECTION or validator.is_valid_file(validate_rn=False)): if self.verbose: click.secho('The files are valid', fg='green') return SUCCESS_RETURN_CODE else: if self.verbose: click.secho('The files are invalid', fg='red') return ERROR_RETURN_CODE else: if self.verbose: click.secho( f'The file {self.output_file} are not part of content repo, Validator Skipped', fg='yellow') return SKIP_RETURN_CODE
def test_wizard_missing_field(self, repo, capsys, missing_field: str): """ Given A Job object in a repo, with one of the required fields missing When Validating the file Then Ensure the structure validator raises a suitable error """ pack = repo.create_pack() wizard = pack.create_wizard(name='wizard_name') wizard.remove(missing_field) validator = StructureValidator(wizard.path, is_new_file=True) with ChangeCWD(repo.path): assert not validator.is_valid_file() captured = capsys.readouterr().out assert f'Missing the field "{missing_field}" in root' in captured
def validate_added_files(self, added_files, file_type: str = None): # noqa: C901 """Validate the added files from your branch. In case we encounter an invalid file we set the self._is_valid param to False. Args: added_files (set): A set of the modified files in the current branch. file_type (str): Used only with -p flag (the type of the file). """ for file_path in added_files: print('Validating {}'.format(file_path)) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE) and not file_type: continue elif 'README' in file_path: readme_validator = ReadMeValidator(file_path) if not readme_validator.is_valid_file(): self._is_valid = False continue structure_validator = StructureValidator( file_path, is_new_file=True, predefined_scheme=file_type) if not structure_validator.is_valid_file(): self._is_valid = False if self.validate_id_set: if not self.id_set_validator.is_file_valid_in_set(file_path): self._is_valid = False if self.id_set_validator.is_file_has_used_id(file_path): self._is_valid = False elif re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE) or file_type == 'playbook': playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook(): self._is_valid = False elif checked_type( file_path, YML_INTEGRATION_REGEXES) or file_type == 'integration': image_validator = ImageValidator(file_path) # if file_type(non git path) the image is not in a separate path image_validator.file_path = file_path if file_type else image_validator.file_path if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if not integration_validator.is_valid_file( validate_rn=not file_type): self._is_valid = False elif checked_type( file_path, PACKAGE_SCRIPTS_REGEXES) or file_type == 'script': unifier = Unifier(os.path.dirname(file_path)) yml_path, _ = unifier.get_script_package_data() # Set file path to the yml file structure_validator.file_path = yml_path script_validator = ScriptValidator(structure_validator) if not script_validator.is_valid_file( validate_rn=not file_type): self._is_valid = False elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): description_validator = DescriptionValidator(file_path) if not description_validator.is_valid_beta_description(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if not integration_validator.is_valid_beta_integration(): self._is_valid = False elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False # incident fields and indicator fields are using the same scheme. elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS) or \ file_type in ('incidentfield', 'indicatorfield'): incident_field_validator = IncidentFieldValidator( structure_validator) if not incident_field_validator.is_valid_file( validate_rn=not file_type): self._is_valid = False elif checked_type(file_path, [REPUTATION_REGEX]) or file_type == 'reputation': reputation_validator = ReputationValidator(structure_validator) if not reputation_validator.is_valid_file( validate_rn=not file_type): self._is_valid = False elif checked_type( file_path, JSON_ALL_LAYOUT_REGEXES) or file_type == 'layout': layout_validator = LayoutValidator(structure_validator) if not layout_validator.is_valid_layout( validate_rn=not file_type): self._is_valid = False elif checked_type( file_path, JSON_ALL_DASHBOARDS_REGEXES) or file_type == 'dashboard': dashboard_validator = DashboardValidator(structure_validator) if not dashboard_validator.is_valid_dashboard( validate_rn=not file_type): self._is_valid = False elif checked_type(file_path, JSON_ALL_INCIDENT_TYPES_REGEXES): incident_type_validator = IncidentTypeValidator( structure_validator) if not incident_type_validator.is_valid_incident_type( validate_rn=not file_type): self._is_valid = False elif 'CHANGELOG' in file_path: self.is_valid_release_notes(file_path) elif checked_type(file_path, CHECKED_TYPES_REGEXES): pass else: print_error( "The file type of {} is not supported in validate command". format(file_path)) print_error( "validate command supports: Integrations, Scripts, Playbooks, " "Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions" ) self._is_valid = False
def validate_modified_files(self, modified_files): # noqa: C901 """Validate the modified files from your branch. In case we encounter an invalid file we set the self._is_valid param to False. Args: modified_files (set): A set of the modified files in the current branch. """ for file_path in modified_files: old_file_path = None if isinstance(file_path, tuple): old_file_path, file_path = file_path print('Validating {}'.format(file_path)) if not checked_type(file_path): print_warning( '- Skipping validation of non-content entity file.') continue if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): continue elif 'README' in file_path: readme_validator = ReadMeValidator(file_path) if not readme_validator.is_valid_file(): self._is_valid = False continue structure_validator = StructureValidator( file_path, old_file_path=old_file_path) if not structure_validator.is_valid_file(): self._is_valid = False if self.validate_id_set: if not self.id_set_validator.is_file_valid_in_set(file_path): self._is_valid = False elif checked_type(file_path, YML_INTEGRATION_REGEXES): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if self.is_backward_check and not integration_validator.is_backward_compatible( ): self._is_valid = False if not integration_validator.is_valid_file(): self._is_valid = False elif checked_type(file_path, YML_BETA_INTEGRATIONS_REGEXES): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid_beta_description(): self._is_valid = False integration_validator = IntegrationValidator( structure_validator) if not integration_validator.is_valid_beta_integration(): self._is_valid = False elif checked_type(file_path, [SCRIPT_REGEX]): script_validator = ScriptValidator(structure_validator) if self.is_backward_check and not script_validator.is_backward_compatible( ): self._is_valid = False if not script_validator.is_valid_file(): self._is_valid = False elif checked_type(file_path, PLAYBOOKS_REGEXES_LIST): playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook( is_new_playbook=False): self._is_valid = False elif checked_type(file_path, PACKAGE_SCRIPTS_REGEXES): unifier = Unifier(os.path.dirname(file_path)) yml_path, _ = unifier.get_script_package_data() # Set file path to the yml file structure_validator.file_path = yml_path script_validator = ScriptValidator(structure_validator) if self.is_backward_check and not script_validator.is_backward_compatible( ): self._is_valid = False if not script_validator.is_valid_file(): self._is_valid = False elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False # incident fields and indicator fields are using the same scheme. elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS): incident_field_validator = IncidentFieldValidator( structure_validator) if not incident_field_validator.is_valid_file( validate_rn=True): self._is_valid = False if self.is_backward_check and not incident_field_validator.is_backward_compatible( ): self._is_valid = False elif checked_type(file_path, [REPUTATION_REGEX]): reputation_validator = ReputationValidator(structure_validator) if not reputation_validator.is_valid_file(validate_rn=True): self._is_valid = False elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES): layout_validator = LayoutValidator(structure_validator) if not layout_validator.is_valid_layout(validate_rn=True): self._is_valid = False elif checked_type(file_path, JSON_ALL_DASHBOARDS_REGEXES): dashboard_validator = DashboardValidator(structure_validator) if not dashboard_validator.is_valid_dashboard( validate_rn=True): self._is_valid = False elif checked_type(file_path, JSON_ALL_INCIDENT_TYPES_REGEXES): incident_type_validator = IncidentTypeValidator( structure_validator) if not incident_type_validator.is_valid_incident_type( validate_rn=True): self._is_valid = False if self.is_backward_check and not incident_type_validator.is_backward_compatible( ): self._is_valid = False elif 'CHANGELOG' in file_path: self.is_valid_release_notes(file_path) elif checked_type(file_path, CHECKED_TYPES_REGEXES): pass else: print_error( "The file type of {} is not supported in validate command". format(file_path)) print_error( "'validate' command supports: Integrations, Scripts, Playbooks, " "Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions" ) self._is_valid = False
def run_all_validations_on_file(self, file_path: str, file_type: str = None) -> None: """ Runs all validations on file specified in 'file_path' Args: file_path: A relative content path to a file to be validated file_type: The output of 'find_type' method """ file_extension = os.path.splitext(file_path)[-1] # We validate only yml json and .md files if file_extension not in ['.yml', '.json', '.md']: return # Ignoring changelog and description files since these are checked on the integration validation if 'changelog' in file_path.lower( ) or 'description' in file_path.lower(): return # unified files should not be validated if file_path.endswith('_unified.yml'): return print(f'Validating {file_path}') if 'README' in file_path: readme_validator = ReadMeValidator(file_path) if not readme_validator.is_valid_file(): self._is_valid = False return structure_validator = StructureValidator(file_path, predefined_scheme=file_type) if not structure_validator.is_valid_file(): self._is_valid = False elif re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook(): self._is_valid = False elif re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE) or file_type == 'playbook': playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook(validate_rn=False): self._is_valid = False elif checked_type(file_path, INTEGRATION_REGXES) or file_type == 'integration': integration_validator = IntegrationValidator(structure_validator) if not integration_validator.is_valid_file(validate_rn=False): self._is_valid = False elif checked_type(file_path, YML_ALL_SCRIPTS_REGEXES) or file_type == 'script': # Set file path to the yml file structure_validator.file_path = file_path script_validator = ScriptValidator(structure_validator) if not script_validator.is_valid_file(validate_rn=False): self._is_valid = False elif checked_type(file_path, YML_BETA_INTEGRATIONS_REGEXES ) or file_type == 'betaintegration': integration_validator = IntegrationValidator(structure_validator) if not integration_validator.is_valid_beta_integration(): self._is_valid = False # incident fields and indicator fields are using the same scheme. elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS) or \ file_type in ('incidentfield', 'indicatorfield'): incident_field_validator = IncidentFieldValidator( structure_validator) if not incident_field_validator.is_valid_file(validate_rn=False): self._is_valid = False elif checked_type( file_path, JSON_ALL_INDICATOR_TYPES_REGEXES) or file_type == 'reputation': reputation_validator = ReputationValidator(structure_validator) if not reputation_validator.is_valid_file(validate_rn=False): self._is_valid = False elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES) or file_type == 'layout': layout_validator = LayoutValidator(structure_validator) if not layout_validator.is_valid_layout(validate_rn=False): self._is_valid = False elif checked_type( file_path, JSON_ALL_DASHBOARDS_REGEXES) or file_type == 'dashboard': dashboard_validator = DashboardValidator(structure_validator) if not dashboard_validator.is_valid_dashboard(validate_rn=False): self._is_valid = False elif checked_type(file_path, JSON_ALL_INCIDENT_TYPES_REGEXES ) or file_type == 'incidenttype': incident_type_validator = IncidentTypeValidator( structure_validator) if not incident_type_validator.is_valid_incident_type( validate_rn=False): self._is_valid = False elif checked_type(file_path, CHECKED_TYPES_REGEXES): print(f'Could not find validations for file {file_path}') else: print_error( 'The file type of {} is not supported in validate command'. format(file_path)) print_error( 'validate command supports: Integrations, Scripts, Playbooks, dashboards, incident types, ' 'reputations, Incident fields, Indicator fields, Images, Release notes, ' 'Layouts and Descriptions') self._is_valid = False
def validate_added_files(self, added_files): # noqa: C901 """Validate the added files from your branch. In case we encounter an invalid file we set the self._is_valid param to False. Args: added_files (set): A set of the modified files in the current branch. """ for file_path in added_files: print('Validating {}'.format(file_path)) if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE): continue structure_validator = StructureValidator(file_path, is_new_file=True) if not structure_validator.is_valid_file(): self._is_valid = False if self.validate_id_set: if not self.id_set_validator.is_file_valid_in_set(file_path): self._is_valid = False if self.id_set_validator.is_file_has_used_id(file_path): self._is_valid = False elif re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE): playbook_validator = PlaybookValidator(structure_validator) if not playbook_validator.is_valid_playbook(): self._is_valid = False elif checked_type(file_path, YML_INTEGRATION_REGEXES): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False description_validator = DescriptionValidator(file_path) if not description_validator.is_valid(): self._is_valid = False integration_validator = IntegrationValidator(structure_validator) if not integration_validator.is_valid_file(validate_rn=False): self._is_valid = False elif checked_type(file_path, PACKAGE_SCRIPTS_REGEXES): unifier = Unifier(os.path.dirname(file_path)) yml_path, _ = unifier.get_script_package_data() # Set file path to the yml file structure_validator.file_path = yml_path script_validator = ScriptValidator(structure_validator) if not script_validator.is_valid_file(validate_rn=False): self._is_valid = False elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \ re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE): description_validator = DescriptionValidator(file_path) if not description_validator.is_valid_beta_description(): self._is_valid = False integration_validator = IntegrationValidator(structure_validator) if not integration_validator.is_valid_beta_integration(): self._is_valid = False elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE): image_validator = ImageValidator(file_path) if not image_validator.is_valid(): self._is_valid = False # incident fields and indicator fields are using the same scheme. elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS): incident_field_validator = IncidentFieldValidator(structure_validator) if not incident_field_validator.is_valid_file(): self._is_valid = False elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES): layout_validator = LayoutValidator(structure_validator) if not layout_validator.is_valid_layout(): self._is_valid = False elif 'CHANGELOG' in file_path: self.is_valid_release_notes(file_path) elif checked_type(file_path, [REPUTATION_REGEX]): print_color( F'Skipping validation for file {file_path} since no validation is currently defined.', LOG_COLORS.YELLOW) elif checked_type(file_path, CHECKED_TYPES_REGEXES): pass else: print_error("The file type of {} is not supported in validate command".format(file_path)) print_error("validate command supports: Integrations, Scripts, Playbooks, " "Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions") self._is_valid = False
def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified=False, old_file_path=None, modified_files=None, added_files=None): """Choose a validator to run for a single file. (i) Args: modified_files: A set of modified files - used for RN validation added_files: A set of added files - used for RN validation old_file_path: The old file path for renamed files pack_error_ignore_list: A dictionary of all pack ignored errors file_path: the file on which to run. is_modified: whether the file is modified or added. Returns: bool. true if file is valid, false otherwise. """ file_type = find_type(file_path) if file_type in self.skipped_file_types: self.ignored_files.add(file_path) return True elif file_type is None: error_message, error_code = Errors.file_type_not_supported() if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path, drop_line=True): return False if not self.check_only_schema: click.echo(f"\nValidating {file_path} as {file_type.value}") structure_validator = StructureValidator(file_path, predefined_scheme=file_type, ignored_errors=pack_error_ignore_list, print_as_warnings=self.print_ignored_errors, tag=self.prev_ver, old_file_path=old_file_path) click.secho(f'Validating scheme for {file_path}') if not structure_validator.is_valid_file(): return False elif self.check_only_schema: return True if self.validate_in_id_set: click.echo(f"Validating id set registration for {file_path}") if not self.id_set_validator.is_file_valid_in_set(file_path): return False # Note: these file are not ignored but there are no additional validators for reports nor connections if file_type in {FileType.REPORT, FileType.CONNECTION}: return True elif file_type == FileType.RELEASE_NOTES: if not self.skip_pack_rn_validation: return self.validate_release_notes(file_path, added_files, modified_files, pack_error_ignore_list, is_modified) elif file_type == FileType.README: return self.validate_readme(file_path, pack_error_ignore_list) elif file_type == FileType.PLAYBOOK: return self.validate_playbook(structure_validator, pack_error_ignore_list) elif file_type == FileType.INTEGRATION: return self.validate_integration(structure_validator, pack_error_ignore_list, is_modified) elif file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT): return self.validate_script(structure_validator, pack_error_ignore_list, is_modified) elif file_type == FileType.BETA_INTEGRATION: return self.validate_beta_integration(structure_validator, pack_error_ignore_list) elif file_type == FileType.IMAGE: return self.validate_image(file_path, pack_error_ignore_list) # incident fields and indicator fields are using the same scheme. elif file_type in (FileType.INCIDENT_FIELD, FileType.INDICATOR_FIELD): return self.validate_incident_field(structure_validator, pack_error_ignore_list, is_modified) elif file_type == FileType.REPUTATION: return self.validate_reputation(structure_validator, pack_error_ignore_list) elif file_type == FileType.LAYOUT: return self.validate_layout(structure_validator, pack_error_ignore_list) elif file_type == FileType.LAYOUTS_CONTAINER: return self.validate_layoutscontainer(structure_validator, pack_error_ignore_list) elif file_type == FileType.DASHBOARD: return self.validate_dashboard(structure_validator, pack_error_ignore_list) elif file_type == FileType.INCIDENT_TYPE: return self.validate_incident_type(structure_validator, pack_error_ignore_list, is_modified) elif file_type == FileType.MAPPER: return self.validate_mapper(structure_validator, pack_error_ignore_list) elif file_type in (FileType.OLD_CLASSIFIER, FileType.CLASSIFIER): return self.validate_classifier(structure_validator, pack_error_ignore_list, file_type) elif file_type == FileType.WIDGET: return self.validate_widget(structure_validator, pack_error_ignore_list) else: error_message, error_code = Errors.file_type_not_supported() if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path): return False return True