class FilesValidator(object):
"""FilesValidator is a class that's designed to validate all the changed files on your branch, and all files in case
you are on master, this class will be used on your local env as the validation hook(pre-commit), and on CircleCi
to make sure you did not bypass the hooks as a safety precaution.
Attributes:
_is_valid (bool): saves the status of the whole validation(instead of mingling it between all the functions).
is_circle (bool): whether we are running on circle or local env.
conf_json_validator (ConfJsonValidator): object for validating the conf.json file.
id_set_validator (IDSetValidator): object for validating the id_set.json file(Created in Circle only).
"""
def __init__(self, is_circle=False):
self._is_valid = True
self.is_circle = is_circle
self.conf_json_validator = ConfJsonValidator()
self.id_set_validator = IDSetValidator(is_circle)
@staticmethod
def is_py_script_or_integration(file_path):
file_yml = get_yaml(file_path)
if re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE):
if file_yml.get('script', {}).get('type',
'javascript') != 'python':
return False
return True
elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
if file_yml.get('type', 'javascript') != 'python':
return False
return True
return False
@staticmethod
def get_modified_files(files_string):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
old_format_files = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower(
) != 'd' and not file_path.endswith('_test.py'):
# naming convention - code file and yml file in packages must have same name.
file_path = os.path.splitext(file_path)[0] + '.yml'
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \
FilesValidator.is_py_script_or_integration(file_path):
old_format_files.add(file_path)
elif file_status.lower() == 'm' and checked_type(
file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(
file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(
file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(
file_path):
modified_files_list.add((file_data[1], file_data[2]))
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(file_path +
" file status is an unknown known one, "
"please check. File status was: " + file_status)
return modified_files_list, added_files_list, deleted_files, old_format_files
def get_modified_and_added_files(self, branch_name, is_circle):
"""Get lists of the modified and added files in your branch according to the git diff output.
Args:
branch_name (string): The name of the branch we are working on.
is_circle (bool): Whether we are running on circle or local env.
Returns:
(modified_files, added_files). Tuple of sets.
"""
all_changed_files_string = run_command(
"git diff --name-status origin/master...{}".format(branch_name))
modified_files, added_files, _, old_format_files = self.get_modified_files(
all_changed_files_string)
if not is_circle:
files_string = run_command(
"git diff --name-status --no-merges HEAD")
non_committed_modified_files, non_committed_added_files, non_committed_deleted_files, \
non_committed_old_format_files = self.get_modified_files(files_string)
all_changed_files_string = run_command(
"git diff --name-status origin/master")
modified_files_from_master, added_files_from_master, _, _ = \
self.get_modified_files(all_changed_files_string)
old_format_files = old_format_files.union(
non_committed_old_format_files)
for mod_file in modified_files_from_master:
if mod_file in non_committed_modified_files:
modified_files.add(mod_file)
for add_file in added_files_from_master:
if add_file in non_committed_added_files:
added_files.add(add_file)
modified_files = modified_files - set(non_committed_deleted_files)
added_files = added_files - set(
non_committed_modified_files) - set(
non_committed_deleted_files)
# new_added_files = set([])
# for added_file in added_files:
# if added_file in non_committed_added_files:
# new_added_files.add(added_file)
# added_files = new_added_files
return modified_files, added_files, old_format_files
def validate_modified_files(self, modified_files, is_backward_check=True):
"""Validate the modified files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
modified_files (set): A set of the modified files in the current branch.
"""
for file_path in modified_files:
old_file_path = None
if isinstance(file_path, tuple):
old_file_path, file_path = file_path
print("Validating {}".format(file_path))
structure_validator = StructureValidator(
file_path,
is_added_file=not (False or is_backward_check),
is_renamed=True if old_file_path else False)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(
file_path, old_file_path=old_file_path)
if is_backward_check and not integration_validator.is_backward_compatible(
):
self._is_valid = False
elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
script_validator = ScriptValidator(file_path)
if is_backward_check and not script_validator.is_backward_compatible(
):
self._is_valid = False
elif re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_JS_REGEX, file_path, re.IGNORECASE):
yml_path, _ = get_script_package_data(
os.path.dirname(file_path))
script_validator = ScriptValidator(yml_path)
if is_backward_check and not script_validator.is_backward_compatible(
):
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
def validate_added_files(self, added_files):
"""Validate the added files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
added_files (set): A set of the modified files in the current branch.
"""
for file_path in added_files:
print("Validating {}".format(file_path))
structure_validator = StructureValidator(file_path,
is_added_file=True)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
if self.id_set_validator.is_file_has_used_id(file_path):
self._is_valid = False
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
if not self.conf_json_validator.is_test_in_conf_json(
collect_ids(file_path)):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
def validate_no_secrets_found(self, branch_name):
"""Check if any secrets are found in your change set.
Args:
branch_name (string): The name of the branch you are working on.
"""
secrets_found = get_secrets(branch_name, self.is_circle)
if secrets_found:
self._is_valid = False
def validate_no_old_format(self, old_format_files):
""" Validate there are no files in the old format(unified yml file for the code and configuration).
Args:
old_format_files(set): file names which are in the old format.
"""
invalid_files = []
for f in old_format_files:
yaml_data = get_yaml(f)
if 'toversion' not in yaml_data: # we only fail on old format if no toversion (meaning it is latest)
invalid_files.append(f)
if invalid_files:
print_error(
"You must update the following files to the new package format. The files are:\n{}"
.format('\n'.join(list(invalid_files))))
self._is_valid = False
def validate_committed_files(self, branch_name, is_backward_check=True):
"""Validate that all the committed files in your branch are valid
Args:
branch_name (string): The name of the branch you are working on.
"""
modified_files, added_files, old_format_files = self.get_modified_and_added_files(
branch_name, self.is_circle)
self.validate_no_secrets_found(branch_name)
self.validate_modified_files(modified_files, is_backward_check)
self.validate_added_files(added_files)
self.validate_no_old_format(old_format_files)
def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
for regex in CHECKED_TYPES_REGEXES:
splitted_regex = regex.split(".*")
directory = splitted_regex[0]
for root, dirs, files in os.walk(directory):
if root not in DIR_LIST: # Skipping in case we entered a package
continue
print_color("Validating {} directory:".format(directory),
LOG_COLORS.GREEN)
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if file_name.startswith('.'):
continue
print("Validating " + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
if root in PACKAGE_SUPPORTING_DIRECTORIES:
for inner_dir in dirs:
file_path = glob.glob(
os.path.join(root, inner_dir, '*.yml'))[0]
print("Validating " + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def is_valid_structure(self, branch_name, is_backward_check=True):
"""Check if the structure is valid for the case we are in, master - all files, branch - changed files.
Args:
branch_name (string): The name of the branch we are working on.
Returns:
(bool). Whether the structure is valid or not.
"""
if not self.conf_json_validator.is_valid_conf_json():
self._is_valid = False
if branch_name != 'master' and not branch_name.startswith(
'19.') and not branch_name.startswith('20.'):
# validates only committed files
self.validate_committed_files(branch_name,
is_backward_check=is_backward_check)
else:
# validates all of Content repo directories according to their schemas
self.validate_all_files()
return self._is_valid
class FilesValidator(object):
"""FilesValidator is a class that's designed to validate all the changed files on your branch, and all files in case
you are on master, this class will be used on your local env as the validation hook(pre-commit), and on CircleCi
to make sure you did not bypass the hooks as a safety precaution.
Attributes:
_is_valid (bool): saves the status of the whole validation(instead of mingling it between all the functions).
is_circle (bool): whether we are running on circle or local env.
print_ignored_files (bool): should print ignored files when iterating over changed files.
conf_json_validator (ConfJsonValidator): object for validating the conf.json file.
id_set_validator (IDSetValidator): object for validating the id_set.json file(Created in Circle only).
"""
def __init__(self, is_circle=False, print_ignored_files=False):
self._is_valid = True
self.is_circle = is_circle
self.print_ignored_files = print_ignored_files
self.conf_json_validator = ConfJsonValidator()
self.id_set_validator = IDSetValidator(is_circle)
@staticmethod
def is_py_script_or_integration(file_path):
file_yml = get_yaml(file_path)
if checked_type(file_path, [INTEGRATION_REGEX, INTEGRATION_YML_REGEX, BETA_INTEGRATION_REGEX,
BETA_INTEGRATION_YML_REGEX]):
return file_yml.get('script', {}).get('type', '') == 'python'
if checked_type(file_path, [SCRIPT_YML_REGEX, SCRIPT_REGEX, SCRIPT_PY_REGEX]):
return file_yml.get('type', '') == 'python'
return False
@staticmethod
def get_modified_files(files_string, tag='master', print_ignored_files=False):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
tag (string): String of git tag used to update modified files.
print_ignored_files (bool): should print ignored files.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
old_format_files = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower() != 'd' \
and not file_path.endswith('_test.py'):
# naming convention - code file and yml file in packages must have same name.
file_path = os.path.splitext(file_path)[0] + '.yml'
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \
FilesValidator.is_py_script_or_integration(file_path):
old_format_files.add(file_path)
elif file_status.lower() == 'm' and checked_type(file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(file_path):
# if a code file changed, take the associated yml file.
if checked_type(file_data[2], CODE_FILES_REGEX):
modified_files_list.add(file_path)
else:
modified_files_list.add((file_data[1], file_data[2]))
elif checked_type(file_path, [SCHEMA_REGEX]):
modified_files_list.add(file_path)
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error('{} file status is an unknown known one, please check. File status was: {}'.format(
file_path, file_status))
elif print_ignored_files and not checked_type(file_path, IGNORED_TYPES_REGEXES):
print_warning('Ignoring file path: {}'.format(file_path))
modified_files_list, added_files_list, deleted_files = filter_packagify_changes(
modified_files_list,
added_files_list,
deleted_files,
tag)
return modified_files_list, added_files_list, deleted_files, old_format_files
def get_modified_and_added_files(self, branch_name, is_circle, tag='origin/master'):
"""Get lists of the modified and added files in your branch according to the git diff output.
Args:
branch_name (string): The name of the branch we are working on.
is_circle (bool): Whether we are running on circle or local env.
tag (string): String of git tag used to update modified files
Returns:
(modified_files, added_files). Tuple of sets.
"""
# Two dots is the default in git diff, it will compare with the last known commit as the base
# Three dots will compare with the last known shared commit as the base
compare_type = '.' if 'master' in tag else ''
all_changed_files_string = run_command(
'git diff --name-status {tag}..{compare_type}refs/heads/{branch}'.format(tag=tag,
branch=branch_name,
compare_type=compare_type))
modified_files, added_files, _, old_format_files = self.get_modified_files(
all_changed_files_string,
tag=tag,
print_ignored_files=self.print_ignored_files)
if not is_circle:
files_string = run_command('git diff --name-status --no-merges HEAD')
non_committed_modified_files, non_committed_added_files, non_committed_deleted_files, \
non_committed_old_format_files = self.get_modified_files(files_string,
print_ignored_files=self.print_ignored_files)
all_changed_files_string = run_command('git diff --name-status {}'.format(tag))
modified_files_from_tag, added_files_from_tag, _, _ = \
self.get_modified_files(all_changed_files_string, print_ignored_files=self.print_ignored_files)
old_format_files = old_format_files.union(non_committed_old_format_files)
modified_files = modified_files.union(
modified_files_from_tag.intersection(non_committed_modified_files))
added_files = added_files.union(
added_files_from_tag.intersection(non_committed_added_files))
modified_files = modified_files - set(non_committed_deleted_files)
added_files = added_files - set(non_committed_modified_files) - set(non_committed_deleted_files)
# new_added_files = set([])
# for added_file in added_files:
# if added_file in non_committed_added_files:
# new_added_files.add(added_file)
# added_files = new_added_files
return modified_files, added_files, old_format_files
def validate_modified_files(self, modified_files, is_backward_check=True, old_branch='master'):
"""Validate the modified files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
modified_files (set): A set of the modified files in the current branch.
is_backward_check (bool): When set to True will run backward compatibility checks
old_branch (str): Old git branch to compare backward compatibility check to
"""
for file_path in modified_files:
old_file_path = None
if isinstance(file_path, tuple):
old_file_path, file_path = file_path
is_python_file = FilesValidator.is_py_script_or_integration(file_path)
print('Validating {}'.format(file_path))
if not checked_type(file_path):
print_warning('- Skipping validation of non-content entity file.')
continue
structure_validator = StructureValidator(file_path, is_added_file=not (False or is_backward_check),
is_renamed=old_file_path is not None)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(file_path, old_file_path=old_file_path,
old_git_branch=old_branch)
if is_backward_check and not integration_validator.is_backward_compatible():
self._is_valid = False
if not integration_validator.is_valid_integration():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid_beta_description():
self._is_valid = False
integration_validator = IntegrationValidator(file_path, old_file_path=old_file_path)
if not integration_validator.is_valid_beta_integration():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
script_validator = ScriptValidator(file_path, old_file_path=old_file_path, old_git_branch=old_branch)
if is_backward_check and not script_validator.is_backward_compatible():
self._is_valid = False
if not script_validator.is_valid_script():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_JS_REGEX, file_path, re.IGNORECASE):
yml_path, _ = get_script_package_data(os.path.dirname(file_path))
script_validator = ScriptValidator(yml_path, old_file_path=old_file_path, old_git_branch=old_branch)
if is_backward_check and not script_validator.is_backward_compatible():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
elif re.match(INCIDENT_FIELD_REGEX, file_path, re.IGNORECASE):
incident_field_validator = IncidentFieldValidator(file_path, old_file_path=old_file_path,
old_git_branch=old_branch)
if not incident_field_validator.is_valid():
self._is_valid = False
if is_backward_check and not incident_field_validator.is_backward_compatible():
self._is_valid = False
def validate_added_files(self, added_files):
"""Validate the added files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
added_files (set): A set of the modified files in the current branch.
"""
for file_path in added_files:
is_python_file = FilesValidator.is_py_script_or_integration(file_path)
print('Validating {}'.format(file_path))
structure_validator = StructureValidator(file_path, is_added_file=True)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
if self.id_set_validator.is_file_has_used_id(file_path):
self._is_valid = False
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
if not self.conf_json_validator.is_test_in_conf_json(collect_ids(file_path)):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(file_path)
if not integration_validator.is_valid_integration():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=False)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE):
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=False)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid_beta_description():
self._is_valid = False
integration_validator = IntegrationValidator(file_path)
if not integration_validator.is_valid_beta_integration(is_new=True):
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=False)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
elif re.match(INCIDENT_FIELD_REGEX, file_path, re.IGNORECASE):
incident_field_validator = IncidentFieldValidator(file_path)
if not incident_field_validator.is_valid():
self._is_valid = False
def validate_no_old_format(self, old_format_files):
""" Validate there are no files in the old format(unified yml file for the code and configuration).
Args:
old_format_files(set): file names which are in the old format.
"""
invalid_files = []
for f in old_format_files:
yaml_data = get_yaml(f)
if 'toversion' not in yaml_data: # we only fail on old format if no toversion (meaning it is latest)
invalid_files.append(f)
if invalid_files:
print_error('You should update the following files to the package format, for further details please visit '
'https://github.com/demisto/content/tree/master/docs/package_directory_structure. '
'The files are:\n{}'.format('\n'.join(list(invalid_files))))
self._is_valid = False
def validate_committed_files(self, branch_name, is_backward_check=True):
"""Validate that all the committed files in your branch are valid
Args:
branch_name (string): The name of the branch you are working on.
"""
modified_files, added_files, old_format_files = self.get_modified_and_added_files(branch_name, self.is_circle)
schema_changed = False
for f in modified_files:
if isinstance(f, tuple):
_, f = f
if checked_type(f, [SCHEMA_REGEX]):
schema_changed = True
# Ensure schema change did not break BC
if schema_changed:
self.validate_all_files()
else:
self.validate_modified_files(modified_files, is_backward_check)
self.validate_added_files(added_files)
self.validate_no_old_format(old_format_files)
def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
for regex in CHECKED_TYPES_REGEXES:
splitted_regex = regex.split('.*')
directory = splitted_regex[0]
for root, dirs, files in os.walk(directory):
if root not in DIR_LIST: # Skipping in case we entered a package
continue
print_color('Validating {} directory:'.format(directory), LOG_COLORS.GREEN)
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if file_name.startswith('.'):
continue
print('Validating ' + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
if root in PACKAGE_SUPPORTING_DIRECTORIES:
for inner_dir in dirs:
file_path = glob.glob(os.path.join(root, inner_dir, '*.yml'))[0]
print('Validating ' + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def is_valid_structure(self, branch_name, is_backward_check=True, prev_ver=None):
"""Check if the structure is valid for the case we are in, master - all files, branch - changed files.
Args:
branch_name (string): The name of the branch we are working on.
prev_ver (string): The name or SHA1 of the previous content version, which will be validated against.
Returns:
(bool). Whether the structure is valid or not.
"""
if not self.conf_json_validator.is_valid_conf_json():
self._is_valid = False
if branch_name != 'master' and not branch_name.startswith('19.') and not branch_name.startswith('20.'):
# validates only committed files
self.validate_committed_files(branch_name, is_backward_check=is_backward_check)
if not prev_ver:
# validate against master if no version was provided
prev_ver = 'origin/master'
self.validate_against_previous_version(branch_name, prev_ver, no_error=True)
else:
self.validate_against_previous_version(branch_name, prev_ver, no_error=True)
# validates all of Content repo directories according to their schemas
self.validate_all_files()
return self._is_valid
def validate_against_previous_version(self, branch_sha, prev_branch_sha=None, no_error=False):
"""Validate all files that were changed between previous version and branch_sha
Args:
branch_sha (str): Current branch SHA1 to validate
prev_branch_sha (str): Previous branch SHA1 to validate against
no_error (bool): If set to true will restore self._is_valid after run (will not return new errors)
"""
if not prev_branch_sha:
with open('./.circleci/config.yml') as f:
config = yaml.safe_load(f)
prev_branch_sha = config['jobs']['build']['environment']['GIT_SHA1']
print_color('Starting validation against {}'.format(prev_branch_sha), LOG_COLORS.GREEN)
modified_files, _, _ = self.get_modified_and_added_files(branch_sha, self.is_circle, prev_branch_sha)
prev_self_valid = self._is_valid
self.validate_modified_files(modified_files, is_backward_check=True, old_branch=prev_branch_sha)
if no_error:
self._is_valid = prev_self_valid
class FilesValidator(object):
"""FilesValidator is a class that's designed to validate all the changed files on your branch, and all files in case
you are on master, this class will be used on your local env as the validation hook(pre-commit), and on CircleCi
to make sure you did not bypass the hooks as a safety precaution.
Attributes:
_is_valid (bool): saves the status of the whole validation(instead of mingling it between all the functions).
is_circle (bool): whether we are running on circle or local env.
conf_json_validator (ConfJsonValidator): object for validating the conf.json file.
id_set_validator (IDSetValidator): object for validating the id_set.json file(Created in Circle only).
"""
def __init__(self, is_circle=False):
self._is_valid = True
self.is_circle = is_circle
self.conf_json_validator = ConfJsonValidator()
self.id_set_validator = IDSetValidator(is_circle)
@staticmethod
def get_modified_files(files_string):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if checked_type(file_path, CODE_FILES_REGEX):
dir_path = os.path.dirname(file_path)
file_path = glob.glob(dir_path + "/*.yml")[0]
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() == 'm' and checked_type(file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(file_path):
modified_files_list.add((file_data[1], file_data[2]))
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(file_path + " file status is an unknown known one, "
"please check. File status was: " + file_status)
return modified_files_list, added_files_list, deleted_files
def get_modified_and_added_files(self, branch_name, is_circle):
"""Get lists of the modified and added files in your branch according to the git diff output.
Args:
branch_name (string): The name of the branch we are working on.
is_circle (bool): Whether we are running on circle or local env.
Returns:
(modified_files, added_files). Tuple of sets.
"""
all_changed_files_string = run_command("git diff --name-status origin/master...{}".format(branch_name))
modified_files, added_files, _ = self.get_modified_files(all_changed_files_string)
if not is_circle:
files_string = run_command("git diff --name-status --no-merges HEAD")
non_committed_modified_files, non_committed_added_files, non_committed_deleted_files = \
self.get_modified_files(files_string)
all_changed_files_string = run_command("git diff --name-status origin/master")
modified_files_from_master, added_files_from_master, _ = self.get_modified_files(all_changed_files_string)
for mod_file in modified_files_from_master:
if mod_file in non_committed_modified_files:
modified_files.add(mod_file)
for add_file in added_files_from_master:
if add_file in non_committed_added_files:
added_files.add(add_file)
modified_files = modified_files - set(non_committed_deleted_files)
added_files = added_files - set(non_committed_modified_files) - set(non_committed_deleted_files)
new_added_files = set([])
for added_file in added_files:
if added_file in non_committed_added_files:
new_added_files.add(added_file)
added_files = new_added_files
return modified_files, added_files
def validate_modified_files(self, modified_files, is_backward_check=True):
"""Validate the modified files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
modified_files (set): A set of the modified files in the current branch.
"""
for file_path in modified_files:
old_file_path = None
if isinstance(file_path, tuple):
old_file_path, file_path = file_path
print("Validating {}".format(file_path))
structure_validator = StructureValidator(file_path, is_added_file=not(False or is_backward_check),
is_renamed=True if old_file_path else False)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(file_path, old_file_path=old_file_path)
if is_backward_check and not integration_validator.is_backward_compatible():
self._is_valid = False
elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
script_validator = ScriptValidator(file_path)
if is_backward_check and not script_validator.is_backward_compatible():
self._is_valid = False
elif re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_JS_REGEX, file_path, re.IGNORECASE):
yml_path, _ = get_script_package_data(os.path.dirname(file_path))
script_validator = ScriptValidator(yml_path)
if is_backward_check and not script_validator.is_backward_compatible():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
def validate_added_files(self, added_files):
"""Validate the added files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
added_files (set): A set of the modified files in the current branch.
"""
for file_path in added_files:
print("Validating {}".format(file_path))
structure_validator = StructureValidator(file_path, is_added_file=True)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
if self.id_set_validator.is_file_has_used_id(file_path):
self._is_valid = False
if re.match(TEST_PLAYBOOK_REGEX, file_path, re.IGNORECASE):
if not self.conf_json_validator.is_test_in_conf_json(collect_ids(file_path)):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
def validate_no_secrets_found(self, branch_name):
"""Check if any secrets are found in your change set.
Args:
branch_name (string): The name of the branch you are working on.
"""
secrets_found = get_secrets(branch_name, self.is_circle)
if secrets_found:
self._is_valid = False
def validate_committed_files(self, branch_name, is_backward_check=True):
"""Validate that all the committed files in your branch are valid
Args:
branch_name (string): The name of the branch you are working on.
"""
modified_files, added_files = self.get_modified_and_added_files(branch_name, self.is_circle)
self.validate_no_secrets_found(branch_name)
self.validate_modified_files(modified_files, is_backward_check)
self.validate_added_files(added_files)
def validate_all_files(self):
"""Validate all files in the repo are in the right format."""
for regex in CHECKED_TYPES_REGEXES:
splitted_regex = regex.split(".*")
directory = splitted_regex[0]
for root, dirs, files in os.walk(directory):
if root not in DIR_LIST: # Skipping in case we entered a package
continue
print_color("Validating {} directory:".format(directory), LOG_COLORS.GREEN)
for file_name in files:
file_path = os.path.join(root, file_name)
# skipping hidden files
if file_name.startswith('.'):
continue
print("Validating " + file_name)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
if root in PACKAGE_SUPPORTING_DIRECTORIES:
for inner_dir in dirs:
file_path = glob.glob(os.path.join(root, inner_dir, '*.yml'))[0]
print("Validating " + file_path)
structure_validator = StructureValidator(file_path)
if not structure_validator.is_valid_scheme():
self._is_valid = False
def is_valid_structure(self, branch_name, is_backward_check=True):
"""Check if the structure is valid for the case we are in, master - all files, branch - changed files.
Args:
branch_name (string): The name of the branch we are working on.
Returns:
(bool). Whether the structure is valid or not.
"""
if not self.conf_json_validator.is_valid_conf_json():
self._is_valid = False
if branch_name != 'master':
# validates only committed files
self.validate_committed_files(branch_name, is_backward_check=is_backward_check)
else:
# validates all of Content repo directories according to their schemas
self.validate_all_files()
return self._is_valid
