def get_new_and_modified_integrations(git_sha1):
'''Return 2 lists - list of new integrations and list of modified integrations since the commit of the git_sha1.
Args:
git_sha1 (str): The git sha of the commit against which we will run the 'git diff' command.
Returns:
(tuple): Returns a tuple of two lists, the names of the new integrations, and the names of
modified integrations.
'''
# get changed yaml files (filter only added and modified files)
tag = get_last_release_version()
file_validator = FilesValidator()
change_log = run_command('git diff --name-status {}'.format(git_sha1))
modified_files, added_files, removed_files, old_format_files = file_validator.get_modified_files(change_log, tag)
all_integration_regexes = YML_INTEGRATION_REGEXES
all_integration_regexes.extend([INTEGRATION_REGEX, PACKS_INTEGRATION_REGEX, BETA_INTEGRATION_REGEX])
added_integration_files = [
file_path for file_path in added_files if checked_type(file_path, all_integration_regexes)
]
modified_integration_files = [
file_path for file_path in modified_files if
isinstance(file_path, str) and checked_type(file_path, all_integration_regexes)
]
new_integrations_names = [
filepath_to_integration_name(file_path) for
file_path in added_integration_files if filepath_to_integration_name(file_path)
]
modified_integrations_names = [
filepath_to_integration_name(file_path) for
file_path in modified_integration_files if filepath_to_integration_name(file_path)
]
return new_integrations_names, modified_integrations_names
def is_py_script_or_integration(file_path):
file_yml = get_yaml(file_path)
if checked_type(file_path, [INTEGRATION_REGEX, INTEGRATION_YML_REGEX, BETA_INTEGRATION_REGEX,
BETA_INTEGRATION_YML_REGEX]):
return file_yml.get('script', {}).get('type', '') == 'python'
if checked_type(file_path, [SCRIPT_YML_REGEX, SCRIPT_REGEX, SCRIPT_PY_REGEX]):
return file_yml.get('type', '') == 'python'
return False
def get_modified_files(files_string):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
Returns:
(yml_files, md_files). Tuple of sets.
"""
all_files = files_string.split('\n')
yml_files = set([])
md_files = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower().startswith('r'):
file_path = file_data[2]
if file_status.lower() == 'm' or file_status.lower() == 'a' or file_status.lower().startswith('r'):
if checked_type(file_path, SPELLCHECK_FILE_TYPES):
yml_files.add(file_path)
elif re.match(DESCRIPTION_REGEX, file_path, re.IGNORECASE):
md_files.add(file_path)
return yml_files, md_files
def is_valid_version(self):
"""Validate that the version of self.file_path is -1."""
file_extension = os.path.splitext(self.file_path)[1]
version_number = -1
reputations_valid = True
layouts_valid = True
if file_extension == '.yml':
yaml_dict = get_yaml(self.file_path)
version_number = yaml_dict.get('commonfields', {}).get('version')
if not version_number: # some files like playbooks do not have commonfields key
version_number = yaml_dict.get('version')
elif file_extension == '.json':
if checked_type(self.file_path, self.VERSION_SCHEMAS):
file_name = os.path.basename(self.file_path)
json_dict = get_json(self.file_path)
if file_name == "reputations.json":
reputations_valid = self.validate_reputations_file(
json_dict)
elif re.match(LAYOUT_REGEX, self.file_path, re.IGNORECASE):
layouts_valid = self.validate_layout_file(json_dict)
else:
version_number = json_dict.get('version')
if version_number != -1 or not reputations_valid or not layouts_valid:
print_error("The version for our files should always be -1, "
"please update the file {}.".format(self.file_path))
self._is_valid = False
return self._is_valid
def get_related_yml_contents(file_path):
# if script or readme file, search for yml in order to retrieve temp white list
yml_file_contents = ''
# Validate if it is integration documentation file or supported file extension
if checked_type(file_path, REQUIRED_YML_FILE_TYPES):
yml_file_contents = retrieve_related_yml(os.path.dirname(file_path))
return yml_file_contents
def validate_committed_files(self,
branch_name,
is_backward_check=True,
is_forked=False):
"""Validate that all the committed files in your branch are valid
Args:
branch_name (string): The name of the branch you are working on.
"""
modified_files, added_files, old_format_files = self.get_modified_and_added_files(
branch_name, self.is_circle)
schema_changed = False
for f in modified_files:
if isinstance(f, tuple):
_, f = f
if checked_type(f, [SCHEMA_REGEX]):
schema_changed = True
# Ensure schema change did not break BC
if schema_changed:
self.validate_all_files()
else:
if not is_forked:
self.validate_no_secrets_found(branch_name)
self.validate_modified_files(modified_files, is_backward_check)
self.validate_added_files(added_files)
self.validate_no_old_format(old_format_files)
def get_modified_files(files_string):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if checked_type(file_path,
CODE_FILES_REGEX) and file_status.lower() != 'd':
dir_path = os.path.dirname(file_path)
file_path = glob.glob(dir_path + "/*.yml")[0]
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() == 'm' and checked_type(
file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(
file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(
file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(
file_path):
modified_files_list.add((file_data[1], file_data[2]))
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(file_path +
" file status is an unknown known one, "
"please check. File status was: " + file_status)
return modified_files_list, added_files_list, deleted_files
def get_modified_files(files_string, tag='master'):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
tag (string): String of git tag used to update modified files
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
old_format_files = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower() != 'd' \
and not file_path.endswith('_test.py'):
# naming convention - code file and yml file in packages must have same name.
file_path = os.path.splitext(file_path)[0] + '.yml'
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \
FilesValidator.is_py_script_or_integration(file_path):
old_format_files.add(file_path)
elif file_status.lower() == 'm' and checked_type(
file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(
file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(
file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(
file_path):
modified_files_list.add((file_data[1], file_data[2]))
elif checked_type(file_path, [SCHEMA_REGEX]):
modified_files_list.add(file_path)
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(file_path +
" file status is an unknown known one, "
"please check. File status was: " + file_status)
modified_files_list, added_files_list, deleted_files = filter_packagify_changes(
modified_files_list, added_files_list, deleted_files, tag)
return modified_files_list, added_files_list, deleted_files, old_format_files
def get_modified_files(files_string):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if checked_type(file_path, CODE_FILES_REGEX):
dir_path = os.path.dirname(file_path)
file_path = glob.glob(dir_path + "/*.yml")[0]
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() == 'm' and checked_type(file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(file_path):
modified_files_list.add((file_data[1], file_data[2]))
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(file_path + " file status is an unknown known one, "
"please check. File status was: " + file_status)
return modified_files_list, added_files_list, deleted_files
def get_modified_files(files_string):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
old_format_files = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
if checked_type(file_path,
CODE_FILES_REGEX) and file_status.lower() != 'd':
dir_path = os.path.dirname(file_path)
try:
file_path = list(
filter(lambda x: not x.endswith('unified.yml'),
glob.glob(dir_path + "/*.yml")))[0]
except IndexError:
continue
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \
FilesValidator.is_py_script_or_integration(file_path):
old_format_files.add(file_path)
elif file_status.lower() == 'm' and checked_type(
file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(
file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower() == 'd' and checked_type(
file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif file_status.lower().startswith('r') and checked_type(
file_path):
modified_files_list.add((file_data[1], file_data[2]))
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(file_path +
" file status is an unknown known one, "
"please check. File status was: " + file_status)
return modified_files_list, added_files_list, deleted_files, old_format_files
def validate_modified_files(self, modified_files, is_backward_check=True, old_branch='master'):
"""Validate the modified files from your branch.
In case we encounter an invalid file we set the self._is_valid param to False.
Args:
modified_files (set): A set of the modified files in the current branch.
is_backward_check (bool): When set to True will run backward compatibility checks
old_branch (str): Old git branch to compare backward compatibility check to
"""
for file_path in modified_files:
old_file_path = None
if isinstance(file_path, tuple):
old_file_path, file_path = file_path
is_python_file = FilesValidator.is_py_script_or_integration(file_path)
print('Validating {}'.format(file_path))
if not checked_type(file_path):
print_warning('- Skipping validation of non-content entity file.')
continue
structure_validator = StructureValidator(file_path, is_added_file=not (False or is_backward_check),
is_renamed=old_file_path is not None)
if not structure_validator.is_file_valid():
self._is_valid = False
if not self.id_set_validator.is_file_valid_in_set(file_path):
self._is_valid = False
elif re.match(INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid():
self._is_valid = False
integration_validator = IntegrationValidator(file_path, old_file_path=old_file_path,
old_git_branch=old_branch)
if is_backward_check and not integration_validator.is_backward_compatible():
self._is_valid = False
if not integration_validator.is_valid_integration():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(BETA_INTEGRATION_REGEX, file_path, re.IGNORECASE) or \
re.match(BETA_INTEGRATION_YML_REGEX, file_path, re.IGNORECASE):
description_validator = DescriptionValidator(file_path)
if not description_validator.is_valid_beta_description():
self._is_valid = False
integration_validator = IntegrationValidator(file_path, old_file_path=old_file_path)
if not integration_validator.is_valid_beta_integration():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(SCRIPT_REGEX, file_path, re.IGNORECASE):
script_validator = ScriptValidator(file_path, old_file_path=old_file_path, old_git_branch=old_branch)
if is_backward_check and not script_validator.is_backward_compatible():
self._is_valid = False
if not script_validator.is_valid_script():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(SCRIPT_YML_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_PY_REGEX, file_path, re.IGNORECASE) or \
re.match(SCRIPT_JS_REGEX, file_path, re.IGNORECASE):
yml_path, _ = get_script_package_data(os.path.dirname(file_path))
script_validator = ScriptValidator(yml_path, old_file_path=old_file_path, old_git_branch=old_branch)
if is_backward_check and not script_validator.is_backward_compatible():
self._is_valid = False
if is_python_file:
docker_image_validator = DockerImageValidator(file_path, is_modified_file=True)
if not docker_image_validator.is_docker_image_valid():
self._is_valid = False
elif re.match(IMAGE_REGEX, file_path, re.IGNORECASE):
image_validator = ImageValidator(file_path)
if not image_validator.is_valid():
self._is_valid = False
elif re.match(INCIDENT_FIELD_REGEX, file_path, re.IGNORECASE):
incident_field_validator = IncidentFieldValidator(file_path, old_file_path=old_file_path,
old_git_branch=old_branch)
if not incident_field_validator.is_valid():
self._is_valid = False
if is_backward_check and not incident_field_validator.is_backward_compatible():
self._is_valid = False
def verify(acceptable, unacceptable, matched_regex):
for test_path in acceptable:
assert checked_type(test_path, compared_regexes=matched_regex)
for test_path in unacceptable:
assert not checked_type(test_path, compared_regexes=matched_regex)
