def initiate_file_validator(self, validator_type):
""" Run schema validate and file validate of file
Returns:
int 0 in case of success
int 1 in case of error
int 2 in case of skip
"""
if self.no_validate:
print_color(
f'Validator Skipped on file: {self.output_file} , no-validate flag was set.',
LOG_COLORS.YELLOW)
return SKIP_RETURN_CODE
else:
print_color('Starting validating files structure',
LOG_COLORS.GREEN)
if self.relative_content_path:
structure_validator = StructureValidator(
self.relative_content_path)
validator = validator_type(structure_validator)
if structure_validator.is_valid_file(
) and validator.is_valid_file(validate_rn=False):
print_color('The files are valid', LOG_COLORS.GREEN)
return SUCCESS_RETURN_CODE
else:
print_color('The files are invalid', LOG_COLORS.RED)
return ERROR_RETURN_CODE
else:
print_color(
f'The file {self.output_file} are not part of content repo, Validator Skipped',
LOG_COLORS.YELLOW)
return SKIP_RETURN_CODE
def test_is_file_valid(self, source, target, answer):
try:
copyfile(source, target)
structure = StructureValidator(target)
assert structure.is_valid_file() is answer
finally:
os.remove(target)
def test_is_file_valid(self, source, target, answer, mocker):
mocker.patch.object(BaseValidator, 'check_file_flags', return_value='')
try:
copyfile(source, target)
structure = StructureValidator(target)
assert structure.is_valid_file() is answer
finally:
os.remove(target)
def initiate_file_validator(self, validator_type, scheme_type):
print_color('Starting validating files structure', LOG_COLORS.GREEN)
structure = StructureValidator(file_path=str(self.output_file_name),
predefined_scheme=scheme_type)
validator = validator_type(structure)
if structure.is_valid_file() and validator.is_valid_file(
validate_rn=False):
print_color('The files are valid', LOG_COLORS.GREEN)
return 0
else:
print_color('The files are invalid', LOG_COLORS.RED)
return 1
def initiate_file_validator(self, validator_type):
""" Run schema validate and file validate of file
Returns:
int 0 in case of success
int 1 in case of error
int 2 in case of skip
"""
if self.no_validate:
if self.verbose:
click.secho(
f'Validator Skipped on file: {self.output_file} , no-validate flag was set.',
fg='yellow')
return SKIP_RETURN_CODE
else:
if self.verbose:
print_color('Starting validating files structure',
LOG_COLORS.GREEN)
# validates only on files in content repo
if self.relative_content_path:
file_type = find_type(self.output_file)
# validates on the output file generated from the format
structure_validator = StructureValidator(
self.output_file,
predefined_scheme=file_type,
suppress_print=not self.verbose)
validator = validator_type(structure_validator,
suppress_print=not self.verbose)
# TODO: remove the connection condition if we implement a specific validator for connections.
if structure_validator.is_valid_file() and \
(file_type == FileType.CONNECTION or validator.is_valid_file(validate_rn=False)):
if self.verbose:
click.secho('The files are valid', fg='green')
return SUCCESS_RETURN_CODE
else:
if self.verbose:
click.secho('The files are invalid', fg='red')
return ERROR_RETURN_CODE
else:
if self.verbose:
click.secho(
f'The file {self.output_file} are not part of content repo, Validator Skipped',
fg='yellow')
return SKIP_RETURN_CODE
def test_wizard_missing_field(self, repo, capsys, missing_field: str):
"""
Given
A Job object in a repo, with one of the required fields missing
When
Validating the file
Then
Ensure the structure validator raises a suitable error
"""
pack = repo.create_pack()
wizard = pack.create_wizard(name='wizard_name')
wizard.remove(missing_field)
validator = StructureValidator(wizard.path, is_new_file=True)
with ChangeCWD(repo.path):
assert not validator.is_valid_file()
captured = capsys.readouterr().out
assert f'Missing the field "{missing_field}" in root' in captured
def validate_added_files(self,
added_files,
file_type: str = None): # noqa: C901
"""Validate the added files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
added_files (set): A set of the modified files in the current branch.
file_type (str): Used only with -p flag (the type of the file).
"""
for file_path in added_files:
print('Validating {}'.format(file_path))
if re.match(TEST_PLAYBOOK_REGEX, file_path,
re.IGNORECASE) and not file_type:
continue
elif 'README' in file_path:
readme_validator = ReadMeValidator(file_path)
if not readme_validator.is_valid_file():
self._is_valid = False
continue
structure_validator = StructureValidator(
file_path, is_new_file=True, predefined_scheme=file_type)
if not structure_validator.is_valid_file():
self._is_valid = False
if self.validate_id_set:
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
if self.id_set_validator.is_file_has_used_id(file_path):
self._is_valid = False
elif re.match(PLAYBOOK_REGEX, file_path,
re.IGNORECASE) or file_type == 'playbook':
playbook_validator = PlaybookValidator(structure_validator)
if not playbook_validator.is_valid_playbook():
self._is_valid = False
elif checked_type(
file_path,
YML_INTEGRATION_REGEXES) or file_type == 'integration':
image_validator = ImageValidator(file_path)
# if file_type(non git path) the image is not in a separate path
image_validator.file_path = file_path if file_type else image_validator.file_path
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(
structure_validator)
if not integration_validator.is_valid_file(
validate_rn=not file_type):
self._is_valid = False
elif checked_type(
file_path,
PACKAGE_SCRIPTS_REGEXES) or file_type == 'script':
unifier = Unifier(os.path.dirname(file_path))
yml_path, _ = unifier.get_script_package_data()
# Set file path to the yml file
structure_validator.file_path = yml_path
script_validator = ScriptValidator(structure_validator)
if not script_validator.is_valid_file(
validate_rn=not file_type):
self._is_valid = False
elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid_beta_description():
self._is_valid = False
integration_validator = IntegrationValidator(
structure_validator)
if not integration_validator.is_valid_beta_integration():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
# incident fields and indicator fields are using the same scheme.
elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS) or \
file_type in ('incidentfield', 'indicatorfield'):
incident_field_validator = IncidentFieldValidator(
structure_validator)
if not incident_field_validator.is_valid_file(
validate_rn=not file_type):
self._is_valid = False
elif checked_type(file_path,
[REPUTATION_REGEX]) or file_type == 'reputation':
reputation_validator = ReputationValidator(structure_validator)
if not reputation_validator.is_valid_file(
validate_rn=not file_type):
self._is_valid = False
elif checked_type(
file_path,
JSON_ALL_LAYOUT_REGEXES) or file_type == 'layout':
layout_validator = LayoutValidator(structure_validator)
if not layout_validator.is_valid_layout(
validate_rn=not file_type):
self._is_valid = False
elif checked_type(
file_path,
JSON_ALL_DASHBOARDS_REGEXES) or file_type == 'dashboard':
dashboard_validator = DashboardValidator(structure_validator)
if not dashboard_validator.is_valid_dashboard(
validate_rn=not file_type):
self._is_valid = False
elif checked_type(file_path, JSON_ALL_INCIDENT_TYPES_REGEXES):
incident_type_validator = IncidentTypeValidator(
structure_validator)
if not incident_type_validator.is_valid_incident_type(
validate_rn=not file_type):
self._is_valid = False
elif 'CHANGELOG' in file_path:
self.is_valid_release_notes(file_path)
elif checked_type(file_path, CHECKED_TYPES_REGEXES):
pass
else:
print_error(
"The file type of {} is not supported in validate command".
format(file_path))
print_error(
"validate command supports: Integrations, Scripts, Playbooks, "
"Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions"
)
self._is_valid = False
def validate_modified_files(self, modified_files): # noqa: C901
"""Validate the modified files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
modified_files (set): A set of the modified files in the current branch.
"""
for file_path in modified_files:
old_file_path = None
if isinstance(file_path, tuple):
old_file_path, file_path = file_path
print('Validating {}'.format(file_path))
if not checked_type(file_path):
print_warning(
'- Skipping validation of non-content entity file.')
continue
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
continue
elif 'README' in file_path:
readme_validator = ReadMeValidator(file_path)
if not readme_validator.is_valid_file():
self._is_valid = False
continue
structure_validator = StructureValidator(
file_path, old_file_path=old_file_path)
if not structure_validator.is_valid_file():
self._is_valid = False
if self.validate_id_set:
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
elif checked_type(file_path, YML_INTEGRATION_REGEXES):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(
structure_validator)
if self.is_backward_check and not integration_validator.is_backward_compatible(
):
self._is_valid = False
if not integration_validator.is_valid_file():
self._is_valid = False
elif checked_type(file_path, YML_BETA_INTEGRATIONS_REGEXES):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid_beta_description():
self._is_valid = False
integration_validator = IntegrationValidator(
structure_validator)
if not integration_validator.is_valid_beta_integration():
self._is_valid = False
elif checked_type(file_path, [SCRIPT_REGEX]):
script_validator = ScriptValidator(structure_validator)
if self.is_backward_check and not script_validator.is_backward_compatible(
):
self._is_valid = False
if not script_validator.is_valid_file():
self._is_valid = False
elif checked_type(file_path, PLAYBOOKS_REGEXES_LIST):
playbook_validator = PlaybookValidator(structure_validator)
if not playbook_validator.is_valid_playbook(
is_new_playbook=False):
self._is_valid = False
elif checked_type(file_path, PACKAGE_SCRIPTS_REGEXES):
unifier = Unifier(os.path.dirname(file_path))
yml_path, _ = unifier.get_script_package_data()
# Set file path to the yml file
structure_validator.file_path = yml_path
script_validator = ScriptValidator(structure_validator)
if self.is_backward_check and not script_validator.is_backward_compatible(
):
self._is_valid = False
if not script_validator.is_valid_file():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
# incident fields and indicator fields are using the same scheme.
elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS):
incident_field_validator = IncidentFieldValidator(
structure_validator)
if not incident_field_validator.is_valid_file(
validate_rn=True):
self._is_valid = False
if self.is_backward_check and not incident_field_validator.is_backward_compatible(
):
self._is_valid = False
elif checked_type(file_path, [REPUTATION_REGEX]):
reputation_validator = ReputationValidator(structure_validator)
if not reputation_validator.is_valid_file(validate_rn=True):
self._is_valid = False
elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES):
layout_validator = LayoutValidator(structure_validator)
if not layout_validator.is_valid_layout(validate_rn=True):
self._is_valid = False
elif checked_type(file_path, JSON_ALL_DASHBOARDS_REGEXES):
dashboard_validator = DashboardValidator(structure_validator)
if not dashboard_validator.is_valid_dashboard(
validate_rn=True):
self._is_valid = False
elif checked_type(file_path, JSON_ALL_INCIDENT_TYPES_REGEXES):
incident_type_validator = IncidentTypeValidator(
structure_validator)
if not incident_type_validator.is_valid_incident_type(
validate_rn=True):
self._is_valid = False
if self.is_backward_check and not incident_type_validator.is_backward_compatible(
):
self._is_valid = False
elif 'CHANGELOG' in file_path:
self.is_valid_release_notes(file_path)
elif checked_type(file_path, CHECKED_TYPES_REGEXES):
pass
else:
print_error(
"The file type of {} is not supported in validate command".
format(file_path))
print_error(
"'validate' command supports: Integrations, Scripts, Playbooks, "
"Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions"
)
self._is_valid = False
def run_all_validations_on_file(self,
file_path: str,
file_type: str = None) -> None:
"""
Runs all validations on file specified in 'file_path'
Args:
file_path: A relative content path to a file to be validated
file_type: The output of 'find_type' method
"""
file_extension = os.path.splitext(file_path)[-1]
# We validate only yml json and .md files
if file_extension not in ['.yml', '.json', '.md']:
return
# Ignoring changelog and description files since these are checked on the integration validation
if 'changelog' in file_path.lower(
) or 'description' in file_path.lower():
return
# unified files should not be validated
if file_path.endswith('_unified.yml'):
return
print(f'Validating {file_path}')
if 'README' in file_path:
readme_validator = ReadMeValidator(file_path)
if not readme_validator.is_valid_file():
self._is_valid = False
return
structure_validator = StructureValidator(file_path,
predefined_scheme=file_type)
if not structure_validator.is_valid_file():
self._is_valid = False
elif re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
playbook_validator = PlaybookValidator(structure_validator)
if not playbook_validator.is_valid_playbook():
self._is_valid = False
elif re.match(PLAYBOOK_REGEX, file_path,
re.IGNORECASE) or file_type == 'playbook':
playbook_validator = PlaybookValidator(structure_validator)
if not playbook_validator.is_valid_playbook(validate_rn=False):
self._is_valid = False
elif checked_type(file_path,
INTEGRATION_REGXES) or file_type == 'integration':
integration_validator = IntegrationValidator(structure_validator)
if not integration_validator.is_valid_file(validate_rn=False):
self._is_valid = False
elif checked_type(file_path,
YML_ALL_SCRIPTS_REGEXES) or file_type == 'script':
# Set file path to the yml file
structure_validator.file_path = file_path
script_validator = ScriptValidator(structure_validator)
if not script_validator.is_valid_file(validate_rn=False):
self._is_valid = False
elif checked_type(file_path, YML_BETA_INTEGRATIONS_REGEXES
) or file_type == 'betaintegration':
integration_validator = IntegrationValidator(structure_validator)
if not integration_validator.is_valid_beta_integration():
self._is_valid = False
# incident fields and indicator fields are using the same scheme.
elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS) or \
file_type in ('incidentfield', 'indicatorfield'):
incident_field_validator = IncidentFieldValidator(
structure_validator)
if not incident_field_validator.is_valid_file(validate_rn=False):
self._is_valid = False
elif checked_type(
file_path,
JSON_ALL_INDICATOR_TYPES_REGEXES) or file_type == 'reputation':
reputation_validator = ReputationValidator(structure_validator)
if not reputation_validator.is_valid_file(validate_rn=False):
self._is_valid = False
elif checked_type(file_path,
JSON_ALL_LAYOUT_REGEXES) or file_type == 'layout':
layout_validator = LayoutValidator(structure_validator)
if not layout_validator.is_valid_layout(validate_rn=False):
self._is_valid = False
elif checked_type(
file_path,
JSON_ALL_DASHBOARDS_REGEXES) or file_type == 'dashboard':
dashboard_validator = DashboardValidator(structure_validator)
if not dashboard_validator.is_valid_dashboard(validate_rn=False):
self._is_valid = False
elif checked_type(file_path, JSON_ALL_INCIDENT_TYPES_REGEXES
) or file_type == 'incidenttype':
incident_type_validator = IncidentTypeValidator(
structure_validator)
if not incident_type_validator.is_valid_incident_type(
validate_rn=False):
self._is_valid = False
elif checked_type(file_path, CHECKED_TYPES_REGEXES):
print(f'Could not find validations for file {file_path}')
else:
print_error(
'The file type of {} is not supported in validate command'.
format(file_path))
print_error(
'validate command supports: Integrations, Scripts, Playbooks, dashboards, incident types, '
'reputations, Incident fields, Indicator fields, Images, Release notes, '
'Layouts and Descriptions')
self._is_valid = False
def validate_added_files(self, added_files): # noqa: C901
"""Validate the added files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
added_files (set): A set of the modified files in the current branch.
"""
for file_path in added_files:
print('Validating {}'.format(file_path))
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
continue
structure_validator = StructureValidator(file_path, is_new_file=True)
if not structure_validator.is_valid_file():
self._is_valid = False
if self.validate_id_set:
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
if self.id_set_validator.is_file_has_used_id(file_path):
self._is_valid = False
elif re.match(PLAYBOOK_REGEX, file_path, re.IGNORECASE):
playbook_validator = PlaybookValidator(structure_validator)
if not playbook_validator.is_valid_playbook():
self._is_valid = False
elif checked_type(file_path, YML_INTEGRATION_REGEXES):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(structure_validator)
if not integration_validator.is_valid_file(validate_rn=False):
self._is_valid = False
elif checked_type(file_path, PACKAGE_SCRIPTS_REGEXES):
unifier = Unifier(os.path.dirname(file_path))
yml_path, _ = unifier.get_script_package_data()
# Set file path to the yml file
structure_validator.file_path = yml_path
script_validator = ScriptValidator(structure_validator)
if not script_validator.is_valid_file(validate_rn=False):
self._is_valid = False
elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid_beta_description():
self._is_valid = False
integration_validator = IntegrationValidator(structure_validator)
if not integration_validator.is_valid_beta_integration():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
# incident fields and indicator fields are using the same scheme.
elif checked_type(file_path, JSON_INDICATOR_AND_INCIDENT_FIELDS):
incident_field_validator = IncidentFieldValidator(structure_validator)
if not incident_field_validator.is_valid_file():
self._is_valid = False
elif checked_type(file_path, JSON_ALL_LAYOUT_REGEXES):
layout_validator = LayoutValidator(structure_validator)
if not layout_validator.is_valid_layout():
self._is_valid = False
elif 'CHANGELOG' in file_path:
self.is_valid_release_notes(file_path)
elif checked_type(file_path, [REPUTATION_REGEX]):
print_color(
F'Skipping validation for file {file_path} since no validation is currently defined.',
LOG_COLORS.YELLOW)
elif checked_type(file_path, CHECKED_TYPES_REGEXES):
pass
else:
print_error("The file type of {} is not supported in validate command".format(file_path))
print_error("validate command supports: Integrations, Scripts, Playbooks, "
"Incident fields, Indicator fields, Images, Release notes, Layouts and Descriptions")
self._is_valid = False
def run_validations_on_file(self, file_path, pack_error_ignore_list, is_modified=False,
old_file_path=None, modified_files=None, added_files=None):
"""Choose a validator to run for a single file. (i)
Args:
modified_files: A set of modified files - used for RN validation
added_files: A set of added files - used for RN validation
old_file_path: The old file path for renamed files
pack_error_ignore_list: A dictionary of all pack ignored errors
file_path: the file on which to run.
is_modified: whether the file is modified or added.
Returns:
bool. true if file is valid, false otherwise.
"""
file_type = find_type(file_path)
if file_type in self.skipped_file_types:
self.ignored_files.add(file_path)
return True
elif file_type is None:
error_message, error_code = Errors.file_type_not_supported()
if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path,
drop_line=True):
return False
if not self.check_only_schema:
click.echo(f"\nValidating {file_path} as {file_type.value}")
structure_validator = StructureValidator(file_path, predefined_scheme=file_type,
ignored_errors=pack_error_ignore_list,
print_as_warnings=self.print_ignored_errors, tag=self.prev_ver,
old_file_path=old_file_path)
click.secho(f'Validating scheme for {file_path}')
if not structure_validator.is_valid_file():
return False
elif self.check_only_schema:
return True
if self.validate_in_id_set:
click.echo(f"Validating id set registration for {file_path}")
if not self.id_set_validator.is_file_valid_in_set(file_path):
return False
# Note: these file are not ignored but there are no additional validators for reports nor connections
if file_type in {FileType.REPORT, FileType.CONNECTION}:
return True
elif file_type == FileType.RELEASE_NOTES:
if not self.skip_pack_rn_validation:
return self.validate_release_notes(file_path, added_files, modified_files, pack_error_ignore_list,
is_modified)
elif file_type == FileType.README:
return self.validate_readme(file_path, pack_error_ignore_list)
elif file_type == FileType.PLAYBOOK:
return self.validate_playbook(structure_validator, pack_error_ignore_list)
elif file_type == FileType.INTEGRATION:
return self.validate_integration(structure_validator, pack_error_ignore_list, is_modified)
elif file_type in (FileType.SCRIPT, FileType.TEST_SCRIPT):
return self.validate_script(structure_validator, pack_error_ignore_list, is_modified)
elif file_type == FileType.BETA_INTEGRATION:
return self.validate_beta_integration(structure_validator, pack_error_ignore_list)
elif file_type == FileType.IMAGE:
return self.validate_image(file_path, pack_error_ignore_list)
# incident fields and indicator fields are using the same scheme.
elif file_type in (FileType.INCIDENT_FIELD, FileType.INDICATOR_FIELD):
return self.validate_incident_field(structure_validator, pack_error_ignore_list, is_modified)
elif file_type == FileType.REPUTATION:
return self.validate_reputation(structure_validator, pack_error_ignore_list)
elif file_type == FileType.LAYOUT:
return self.validate_layout(structure_validator, pack_error_ignore_list)
elif file_type == FileType.LAYOUTS_CONTAINER:
return self.validate_layoutscontainer(structure_validator, pack_error_ignore_list)
elif file_type == FileType.DASHBOARD:
return self.validate_dashboard(structure_validator, pack_error_ignore_list)
elif file_type == FileType.INCIDENT_TYPE:
return self.validate_incident_type(structure_validator, pack_error_ignore_list, is_modified)
elif file_type == FileType.MAPPER:
return self.validate_mapper(structure_validator, pack_error_ignore_list)
elif file_type in (FileType.OLD_CLASSIFIER, FileType.CLASSIFIER):
return self.validate_classifier(structure_validator, pack_error_ignore_list, file_type)
elif file_type == FileType.WIDGET:
return self.validate_widget(structure_validator, pack_error_ignore_list)
else:
error_message, error_code = Errors.file_type_not_supported()
if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path):
return False
return True
