def main():
arg_parser = argparse.ArgumentParser()
arg_parser.add_argument('version', help='Release version')
arg_parser.add_argument('git_sha1',
help='commit sha1 to compare changes with')
arg_parser.add_argument('asset_id', help='Asset ID')
arg_parser.add_argument('server_version', help='Server version')
arg_parser.add_argument('--github-token', help='Github token')
args = arg_parser.parse_args()
tag = get_last_release_version()
print('Last release version: {}'.format(tag))
# get changed yaml/json files (filter only relevant changed files)
file_validator = FilesValidator()
try:
change_log = run_command('git diff --name-status {}'.format(
args.git_sha1),
exit_on_error=False)
except RuntimeError:
print_error(
'Unable to get the SHA1 of the commit in which the version was released. This can happen if your '
'branch is not updated with origin master. Merge from origin master and, try again.\n'
'If you\'re not on a fork, run "git merge origin/master".\n'
'If you are on a fork, first set https://github.com/demisto/content to be '
'your upstream by running "git remote add upstream https://github.com/demisto/content". After '
'setting the upstream, run "git fetch upstream", and then run "git merge upstream/master". Doing '
'these steps will merge your branch with content master as a base.'
)
sys.exit(1)
else:
modified_files, added_files, removed_files, _ = file_validator.get_modified_files(
change_log)
modified_files, added_files, removed_files = filter_packagify_changes(
modified_files, added_files, removed_files, tag=tag)
for file_path in added_files:
create_file_release_notes('A', file_path)
for file_path in modified_files:
create_file_release_notes('M', file_path)
for file_path in removed_files:
# content entities are only yml/json files. ignore all the rest.
if file_path.endswith('.yml') or file_path.endswith('.json'):
handle_deleted_file(file_path, tag)
# join all release notes
res = []
beta_res = []
missing_release_notes = False
for key in RELEASE_NOTES_ORDER:
value = RELEASE_NOTE_GENERATOR[key]
ans, beta_ans = value.generate_release_notes(args.server_version)
if ans is None or value.is_missing_release_notes:
missing_release_notes = True
if ans:
res.append(ans)
if beta_ans:
beta_res.append(beta_ans)
release_notes = "\n---\n".join(res)
beta_release_notes = "\n---\n".join(beta_res)
create_content_descriptor(args.version,
args.asset_id,
release_notes,
args.github_token,
beta_rn=beta_release_notes)
if missing_release_notes:
print_error(
"Error: some release notes are missing. See previous errors.")
sys.exit(1)
def test_filter_packagify_changes(self, path):
modified, added, removed = filter_packagify_changes(
modified_files=[], added_files=[], removed_files=[path])
assert modified == []
assert added == set()
assert removed == [VALID_MD]
def filter_changed_files(files_string,
tag='master',
print_ignored_files=False):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
tag (string): String of git tag used to update modified files.
print_ignored_files (bool): should print ignored files.
Returns:
Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set()
added_files_list = set()
modified_files_list = set()
old_format_files = set()
changed_meta_files = set()
ignored_files = set()
new_packs = set()
for f in all_files:
file_data: list = list(filter(None, f.split('\t')))
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
try:
file_type = find_type(file_path)
# if the file is a code file - change path to
# the associated yml path to trigger release notes validation.
if file_status.lower() != 'd' and \
file_type in [FileType.POWERSHELL_FILE, FileType.PYTHON_FILE] and \
not (file_path.endswith('_test.py') or file_path.endswith('.Tests.ps1')):
# naming convention - code file and yml file in packages must have same name.
file_path = os.path.splitext(file_path)[0] + '.yml'
# ignore changes in JS files and unit test files.
elif file_path.endswith('.js') or file_path.endswith(
'.py') or file_path.endswith('.ps1'):
if file_path not in ignored_files:
ignored_files.add(file_path)
if print_ignored_files:
click.secho(
'Ignoring file path: {} - code file'.format(
file_path),
fg="yellow")
continue
# ignore changes in TESTS_DIRECTORIES files.
elif any(test_dir in file_path
for test_dir in TESTS_AND_DOC_DIRECTORIES):
if file_path not in ignored_files:
ignored_files.add(file_path)
if print_ignored_files:
click.secho(
'Ignoring file path: {} - test file'.format(
file_path),
fg="yellow")
continue
# identify deleted files
if file_status.lower() == 'd' and not file_path.startswith('.'):
deleted_files.add(file_path)
# ignore directories
elif not os.path.isfile(file_path):
if print_ignored_files:
click.secho(
'Ignoring file path: {} - directory'.format(file_path),
fg="yellow")
continue
# changes in old scripts and integrations - unified python scripts/integrations
elif file_status.lower() in ['m', 'a', 'r'] and \
file_type in [FileType.INTEGRATION, FileType.SCRIPT] and \
ValidateManager.is_old_file_format(file_path, file_type):
old_format_files.add(file_path)
# identify modified files
elif file_status.lower(
) == 'm' and file_type and not file_path.startswith('.'):
modified_files_list.add(file_path)
# identify added files
elif file_status.lower(
) == 'a' and file_type and not file_path.startswith('.'):
added_files_list.add(file_path)
# identify renamed files
elif file_status.lower().startswith('r') and file_type:
# if a code file changed, take the associated yml file.
if file_type in [
FileType.POWERSHELL_FILE, FileType.PYTHON_FILE
]:
modified_files_list.add(file_path)
else:
# file_data[1] = old name, file_data[2] = new name
modified_files_list.add((file_data[1], file_data[2]))
elif file_status.lower() not in KNOWN_FILE_STATUSES:
click.secho(
'{} file status is an unknown one, please check. File status was: {}'
.format(file_path, file_status),
fg="bright_red")
# handle meta data file changes
elif file_path.endswith(PACKS_PACK_META_FILE_NAME):
if file_status.lower() == 'a':
new_packs.add(get_pack_name(file_path))
elif file_status.lower() == 'm':
changed_meta_files.add(file_path)
else:
# pipefile and pipelock files should not enter to ignore_files
if 'Pipfile' not in file_path:
if file_path not in ignored_files:
ignored_files.add(file_path)
if print_ignored_files:
click.secho(
'Ignoring file path: {} - system file'.format(
file_path),
fg="yellow")
else:
if print_ignored_files:
click.secho(
'Ignoring file path: {} - system file'.format(
file_path),
fg="yellow")
# handle a case where a file was deleted locally though recognised as added against master.
except FileNotFoundError:
if file_path not in ignored_files:
ignored_files.add(file_path)
if print_ignored_files:
click.secho(
'Ignoring file path: {} - File not found'.format(
file_path),
fg="yellow")
modified_files_list, added_files_list, deleted_files = filter_packagify_changes(
modified_files_list, added_files_list, deleted_files, tag)
return modified_files_list, added_files_list, deleted_files, old_format_files, \
changed_meta_files, ignored_files, new_packs
def get_modified_files(files_string,
tag='master',
print_ignored_files=False):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
tag (string): String of git tag used to update modified files.
print_ignored_files (bool): should print ignored files.
Returns:
(modified_files_list, added_files_list, deleted_files). Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set([])
added_files_list = set([])
modified_files_list = set([])
old_format_files = set([])
for f in all_files:
file_data = f.split()
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower() != 'd' \
and not file_path.endswith('_test.py'):
# naming convention - code file and yml file in packages must have same name.
file_path = os.path.splitext(file_path)[0] + '.yml'
elif file_path.endswith('.js') or file_path.endswith('.py'):
continue
if file_status.lower() == 'd' and checked_type(
file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
elif not os.path.isfile(file_path):
continue
elif file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \
FilesValidator._is_py_script_or_integration(file_path):
old_format_files.add(file_path)
elif file_status.lower() == 'm' and checked_type(
file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
elif file_status.lower() == 'a' and checked_type(
file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
elif file_status.lower().startswith('r') and checked_type(
file_path):
# if a code file changed, take the associated yml file.
if checked_type(file_data[2], CODE_FILES_REGEX):
modified_files_list.add(file_path)
else:
modified_files_list.add((file_data[1], file_data[2]))
elif checked_type(file_path, [SCHEMA_REGEX]):
modified_files_list.add(file_path)
elif file_status.lower() not in KNOWN_FILE_STATUSES:
print_error(
'{} file status is an unknown one, please check. File status was: {}'
.format(file_path, file_status))
elif print_ignored_files and not checked_type(
file_path, IGNORED_TYPES_REGEXES):
print_warning('Ignoring file path: {}'.format(file_path))
modified_files_list, added_files_list, deleted_files = filter_packagify_changes(
modified_files_list, added_files_list, deleted_files, tag)
return modified_files_list, added_files_list, deleted_files, old_format_files
def filter_changed_files(self, files_string, tag='master', print_ignored_files=False):
"""Get lists of the modified files in your branch according to the files string.
Args:
files_string (string): String that was calculated by git using `git diff` command.
tag (string): String of git tag used to update modified files.
print_ignored_files (bool): should print ignored files.
Returns:
Tuple of sets.
"""
all_files = files_string.split('\n')
deleted_files = set()
added_files_list = set()
modified_files_list = set()
old_format_files = set()
changed_meta_files = set()
for f in all_files:
file_data = list(filter(None, f.split('\t')))
if not file_data:
continue
file_status = file_data[0]
file_path = file_data[1]
if file_status.lower().startswith('r'):
file_status = 'r'
file_path = file_data[2]
# if the file is a code file - change path to the associated yml path.
if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower() != 'd' \
and not (file_path.endswith('_test.py') or file_path.endswith('.Tests.ps1')):
# naming convention - code file and yml file in packages must have same name.
file_path = os.path.splitext(file_path)[0] + '.yml'
# ignore changes in JS files and unit test files.
elif file_path.endswith('.js') or file_path.endswith('.py') or file_path.endswith('.ps1'):
self.ignored_files.add(file_path)
continue
# identify deleted files
if file_status.lower() == 'd' and checked_type(file_path) and not file_path.startswith('.'):
deleted_files.add(file_path)
# ignore directories
elif not os.path.isfile(file_path):
continue
# changes in old scripts and integrations - unified python scripts/integrations
elif file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \
self._is_py_script_or_integration(file_path):
old_format_files.add(file_path)
# identify modified files
elif file_status.lower() == 'm' and checked_type(file_path) and not file_path.startswith('.'):
modified_files_list.add(file_path)
# identify added files
elif file_status.lower() == 'a' and checked_type(file_path) and not file_path.startswith('.'):
added_files_list.add(file_path)
# identify renamed files
elif file_status.lower().startswith('r') and checked_type(file_path):
# if a code file changed, take the associated yml file.
if checked_type(file_data[2], CODE_FILES_REGEX):
modified_files_list.add(file_path)
else:
# file_data[1] = old name, file_data[2] = new name
modified_files_list.add((file_data[1], file_data[2]))
# detect changes in schema
elif checked_type(file_path, [SCHEMA_REGEX]):
modified_files_list.add(file_path)
self.changes_in_schema = True
elif file_status.lower() not in KNOWN_FILE_STATUSES:
click.secho('{} file status is an unknown one, please check. File status was: {}'
.format(file_path, file_status), fg="bright_red")
elif file_path.endswith(PACKS_PACK_META_FILE_NAME):
if file_status.lower() == 'a':
self.new_packs.add(get_pack_name(file_path))
elif file_status.lower() == 'm':
changed_meta_files.add(file_path)
elif print_ignored_files and not checked_type(file_path, IGNORED_TYPES_REGEXES):
if file_path not in self.ignored_files:
self.ignored_files.add(file_path)
click.secho('Ignoring file path: {}'.format(file_path), fg="yellow")
modified_files_list, added_files_list, deleted_files = filter_packagify_changes(
modified_files_list,
added_files_list,
deleted_files,
tag)
return modified_files_list, added_files_list, deleted_files, old_format_files, changed_meta_files