def main(): arg_parser = argparse.ArgumentParser() arg_parser.add_argument('version', help='Release version') arg_parser.add_argument('git_sha1', help='commit sha1 to compare changes with') arg_parser.add_argument('asset_id', help='Asset ID') arg_parser.add_argument('server_version', help='Server version') arg_parser.add_argument('--github-token', help='Github token') args = arg_parser.parse_args() tag = get_last_release_version() print('Last release version: {}'.format(tag)) # get changed yaml/json files (filter only relevant changed files) file_validator = FilesValidator() try: change_log = run_command('git diff --name-status {}'.format( args.git_sha1), exit_on_error=False) except RuntimeError: print_error( 'Unable to get the SHA1 of the commit in which the version was released. This can happen if your ' 'branch is not updated with origin master. Merge from origin master and, try again.\n' 'If you\'re not on a fork, run "git merge origin/master".\n' 'If you are on a fork, first set https://github.com/demisto/content to be ' 'your upstream by running "git remote add upstream https://github.com/demisto/content". After ' 'setting the upstream, run "git fetch upstream", and then run "git merge upstream/master". Doing ' 'these steps will merge your branch with content master as a base.' ) sys.exit(1) else: modified_files, added_files, removed_files, _ = file_validator.get_modified_files( change_log) modified_files, added_files, removed_files = filter_packagify_changes( modified_files, added_files, removed_files, tag=tag) for file_path in added_files: create_file_release_notes('A', file_path) for file_path in modified_files: create_file_release_notes('M', file_path) for file_path in removed_files: # content entities are only yml/json files. ignore all the rest. if file_path.endswith('.yml') or file_path.endswith('.json'): handle_deleted_file(file_path, tag) # join all release notes res = [] beta_res = [] missing_release_notes = False for key in RELEASE_NOTES_ORDER: value = RELEASE_NOTE_GENERATOR[key] ans, beta_ans = value.generate_release_notes(args.server_version) if ans is None or value.is_missing_release_notes: missing_release_notes = True if ans: res.append(ans) if beta_ans: beta_res.append(beta_ans) release_notes = "\n---\n".join(res) beta_release_notes = "\n---\n".join(beta_res) create_content_descriptor(args.version, args.asset_id, release_notes, args.github_token, beta_rn=beta_release_notes) if missing_release_notes: print_error( "Error: some release notes are missing. See previous errors.") sys.exit(1)
def test_filter_packagify_changes(self, path): modified, added, removed = filter_packagify_changes( modified_files=[], added_files=[], removed_files=[path]) assert modified == [] assert added == set() assert removed == [VALID_MD]
def filter_changed_files(files_string, tag='master', print_ignored_files=False): """Get lists of the modified files in your branch according to the files string. Args: files_string (string): String that was calculated by git using `git diff` command. tag (string): String of git tag used to update modified files. print_ignored_files (bool): should print ignored files. Returns: Tuple of sets. """ all_files = files_string.split('\n') deleted_files = set() added_files_list = set() modified_files_list = set() old_format_files = set() changed_meta_files = set() ignored_files = set() new_packs = set() for f in all_files: file_data: list = list(filter(None, f.split('\t'))) if not file_data: continue file_status = file_data[0] file_path = file_data[1] if file_status.lower().startswith('r'): file_status = 'r' file_path = file_data[2] try: file_type = find_type(file_path) # if the file is a code file - change path to # the associated yml path to trigger release notes validation. if file_status.lower() != 'd' and \ file_type in [FileType.POWERSHELL_FILE, FileType.PYTHON_FILE] and \ not (file_path.endswith('_test.py') or file_path.endswith('.Tests.ps1')): # naming convention - code file and yml file in packages must have same name. file_path = os.path.splitext(file_path)[0] + '.yml' # ignore changes in JS files and unit test files. elif file_path.endswith('.js') or file_path.endswith( '.py') or file_path.endswith('.ps1'): if file_path not in ignored_files: ignored_files.add(file_path) if print_ignored_files: click.secho( 'Ignoring file path: {} - code file'.format( file_path), fg="yellow") continue # ignore changes in TESTS_DIRECTORIES files. elif any(test_dir in file_path for test_dir in TESTS_AND_DOC_DIRECTORIES): if file_path not in ignored_files: ignored_files.add(file_path) if print_ignored_files: click.secho( 'Ignoring file path: {} - test file'.format( file_path), fg="yellow") continue # identify deleted files if file_status.lower() == 'd' and not file_path.startswith('.'): deleted_files.add(file_path) # ignore directories elif not os.path.isfile(file_path): if print_ignored_files: click.secho( 'Ignoring file path: {} - directory'.format(file_path), fg="yellow") continue # changes in old scripts and integrations - unified python scripts/integrations elif file_status.lower() in ['m', 'a', 'r'] and \ file_type in [FileType.INTEGRATION, FileType.SCRIPT] and \ ValidateManager.is_old_file_format(file_path, file_type): old_format_files.add(file_path) # identify modified files elif file_status.lower( ) == 'm' and file_type and not file_path.startswith('.'): modified_files_list.add(file_path) # identify added files elif file_status.lower( ) == 'a' and file_type and not file_path.startswith('.'): added_files_list.add(file_path) # identify renamed files elif file_status.lower().startswith('r') and file_type: # if a code file changed, take the associated yml file. if file_type in [ FileType.POWERSHELL_FILE, FileType.PYTHON_FILE ]: modified_files_list.add(file_path) else: # file_data[1] = old name, file_data[2] = new name modified_files_list.add((file_data[1], file_data[2])) elif file_status.lower() not in KNOWN_FILE_STATUSES: click.secho( '{} file status is an unknown one, please check. File status was: {}' .format(file_path, file_status), fg="bright_red") # handle meta data file changes elif file_path.endswith(PACKS_PACK_META_FILE_NAME): if file_status.lower() == 'a': new_packs.add(get_pack_name(file_path)) elif file_status.lower() == 'm': changed_meta_files.add(file_path) else: # pipefile and pipelock files should not enter to ignore_files if 'Pipfile' not in file_path: if file_path not in ignored_files: ignored_files.add(file_path) if print_ignored_files: click.secho( 'Ignoring file path: {} - system file'.format( file_path), fg="yellow") else: if print_ignored_files: click.secho( 'Ignoring file path: {} - system file'.format( file_path), fg="yellow") # handle a case where a file was deleted locally though recognised as added against master. except FileNotFoundError: if file_path not in ignored_files: ignored_files.add(file_path) if print_ignored_files: click.secho( 'Ignoring file path: {} - File not found'.format( file_path), fg="yellow") modified_files_list, added_files_list, deleted_files = filter_packagify_changes( modified_files_list, added_files_list, deleted_files, tag) return modified_files_list, added_files_list, deleted_files, old_format_files, \ changed_meta_files, ignored_files, new_packs
def get_modified_files(files_string, tag='master', print_ignored_files=False): """Get lists of the modified files in your branch according to the files string. Args: files_string (string): String that was calculated by git using `git diff` command. tag (string): String of git tag used to update modified files. print_ignored_files (bool): should print ignored files. Returns: (modified_files_list, added_files_list, deleted_files). Tuple of sets. """ all_files = files_string.split('\n') deleted_files = set([]) added_files_list = set([]) modified_files_list = set([]) old_format_files = set([]) for f in all_files: file_data = f.split() if not file_data: continue file_status = file_data[0] file_path = file_data[1] if file_status.lower().startswith('r'): file_status = 'r' file_path = file_data[2] if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower() != 'd' \ and not file_path.endswith('_test.py'): # naming convention - code file and yml file in packages must have same name. file_path = os.path.splitext(file_path)[0] + '.yml' elif file_path.endswith('.js') or file_path.endswith('.py'): continue if file_status.lower() == 'd' and checked_type( file_path) and not file_path.startswith('.'): deleted_files.add(file_path) elif not os.path.isfile(file_path): continue elif file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \ FilesValidator._is_py_script_or_integration(file_path): old_format_files.add(file_path) elif file_status.lower() == 'm' and checked_type( file_path) and not file_path.startswith('.'): modified_files_list.add(file_path) elif file_status.lower() == 'a' and checked_type( file_path) and not file_path.startswith('.'): added_files_list.add(file_path) elif file_status.lower().startswith('r') and checked_type( file_path): # if a code file changed, take the associated yml file. if checked_type(file_data[2], CODE_FILES_REGEX): modified_files_list.add(file_path) else: modified_files_list.add((file_data[1], file_data[2])) elif checked_type(file_path, [SCHEMA_REGEX]): modified_files_list.add(file_path) elif file_status.lower() not in KNOWN_FILE_STATUSES: print_error( '{} file status is an unknown one, please check. File status was: {}' .format(file_path, file_status)) elif print_ignored_files and not checked_type( file_path, IGNORED_TYPES_REGEXES): print_warning('Ignoring file path: {}'.format(file_path)) modified_files_list, added_files_list, deleted_files = filter_packagify_changes( modified_files_list, added_files_list, deleted_files, tag) return modified_files_list, added_files_list, deleted_files, old_format_files
def filter_changed_files(self, files_string, tag='master', print_ignored_files=False): """Get lists of the modified files in your branch according to the files string. Args: files_string (string): String that was calculated by git using `git diff` command. tag (string): String of git tag used to update modified files. print_ignored_files (bool): should print ignored files. Returns: Tuple of sets. """ all_files = files_string.split('\n') deleted_files = set() added_files_list = set() modified_files_list = set() old_format_files = set() changed_meta_files = set() for f in all_files: file_data = list(filter(None, f.split('\t'))) if not file_data: continue file_status = file_data[0] file_path = file_data[1] if file_status.lower().startswith('r'): file_status = 'r' file_path = file_data[2] # if the file is a code file - change path to the associated yml path. if checked_type(file_path, CODE_FILES_REGEX) and file_status.lower() != 'd' \ and not (file_path.endswith('_test.py') or file_path.endswith('.Tests.ps1')): # naming convention - code file and yml file in packages must have same name. file_path = os.path.splitext(file_path)[0] + '.yml' # ignore changes in JS files and unit test files. elif file_path.endswith('.js') or file_path.endswith('.py') or file_path.endswith('.ps1'): self.ignored_files.add(file_path) continue # identify deleted files if file_status.lower() == 'd' and checked_type(file_path) and not file_path.startswith('.'): deleted_files.add(file_path) # ignore directories elif not os.path.isfile(file_path): continue # changes in old scripts and integrations - unified python scripts/integrations elif file_status.lower() in ['m', 'a', 'r'] and checked_type(file_path, OLD_YML_FORMAT_FILE) and \ self._is_py_script_or_integration(file_path): old_format_files.add(file_path) # identify modified files elif file_status.lower() == 'm' and checked_type(file_path) and not file_path.startswith('.'): modified_files_list.add(file_path) # identify added files elif file_status.lower() == 'a' and checked_type(file_path) and not file_path.startswith('.'): added_files_list.add(file_path) # identify renamed files elif file_status.lower().startswith('r') and checked_type(file_path): # if a code file changed, take the associated yml file. if checked_type(file_data[2], CODE_FILES_REGEX): modified_files_list.add(file_path) else: # file_data[1] = old name, file_data[2] = new name modified_files_list.add((file_data[1], file_data[2])) # detect changes in schema elif checked_type(file_path, [SCHEMA_REGEX]): modified_files_list.add(file_path) self.changes_in_schema = True elif file_status.lower() not in KNOWN_FILE_STATUSES: click.secho('{} file status is an unknown one, please check. File status was: {}' .format(file_path, file_status), fg="bright_red") elif file_path.endswith(PACKS_PACK_META_FILE_NAME): if file_status.lower() == 'a': self.new_packs.add(get_pack_name(file_path)) elif file_status.lower() == 'm': changed_meta_files.add(file_path) elif print_ignored_files and not checked_type(file_path, IGNORED_TYPES_REGEXES): if file_path not in self.ignored_files: self.ignored_files.add(file_path) click.secho('Ignoring file path: {}'.format(file_path), fg="yellow") modified_files_list, added_files_list, deleted_files = filter_packagify_changes( modified_files_list, added_files_list, deleted_files, tag) return modified_files_list, added_files_list, deleted_files, old_format_files, changed_meta_files