def get_modified_files_for_testing(git_diff: str) -> ModifiedFiles: """ Gets git diff string and filters those files into tests: Args: git_diff: a git diff output (with --name-only flag) Returns: ModifiedFiles instance """ types_to_files: Dict[FileType, Set[str]] = create_type_to_file(git_diff) # Mapping of the files FileType: file path # Checks if any common file exists in types_to_file changed_common_files = get_common_files(types_to_files.get(FileType.SCRIPT, set())) types_to_files = remove_common_files(types_to_files, changed_common_files) # Sample tests are the remaining python files sample_tests = types_to_files.get(FileType.PYTHON_FILE, set()) # Modified files = YMLs of integrations, scripts and playbooks modified_files: Set[str] = types_to_files.get(FileType.INTEGRATION, set()).union( types_to_files.get(FileType.SCRIPT, set()), types_to_files.get(FileType.BETA_INTEGRATION, set()), types_to_files.get(FileType.PLAYBOOK, set())) # Modified YMLs for testing (Integrations, Scripts, Playbooks). # Metadata packs modified_metadata: Set[str] = set() for file_path in types_to_files.get(FileType.METADATA, set()): if pack_name := tools.get_pack_name(file_path): modified_metadata.add(pack_name)
def get_packs_with_added_release_notes(added_files): added_rn = set() for file in added_files: if find_type(path=file) == FileType.RELEASE_NOTES: added_rn.add(get_pack_name(file)) return added_rn
def is_valid_file(self, validate_rn=True): # type: (bool) -> bool """Check whether the script is valid or not""" is_script_valid = all([ super().is_valid_file(validate_rn), self.validate_readme_exists(self.validate_all), self.is_valid_subtype(), self.is_id_equals_name(), self.is_docker_image_valid(), self.is_valid_pwsh(), self.is_valid_script_file_path(), self.is_there_separators_in_names(), self.name_not_contain_the_type(), self.runas_is_not_dbtrole(), ]) # check only on added files if not self.old_file: is_script_valid = all([ is_script_valid, self.is_valid_name() ]) core_packs_list = get_core_pack_list() pack = get_pack_name(self.file_path) is_core = True if pack in core_packs_list else False if is_core: is_script_valid = all([ is_script_valid, self.no_incident_in_core_pack() ]) return is_script_valid
def run_validation_on_specific_files(self): """Run validations only on specific files """ files_validation_result = set() for path in self.file_path.split(','): error_ignore_list = self.get_error_ignore_list(get_pack_name(path)) if os.path.isfile(path): click.secho('\n================= Validating file =================', fg="bright_cyan") files_validation_result.add(self.run_validations_on_file(path, error_ignore_list)) else: path = path.rstrip('/') dir_name = os.path.basename(path) if dir_name in CONTENT_ENTITIES_DIRS: click.secho(f'\n================= Validating content directory {path} =================', fg="bright_cyan") files_validation_result.add(self.run_validation_on_content_entities(path, error_ignore_list)) else: if os.path.basename(os.path.dirname(path)) == PACKS_DIR: click.secho(f'\n================= Validating pack {path} =================', fg="bright_cyan") files_validation_result.add(self.run_validations_on_pack(path)) else: click.secho(f'\n================= Validating package {path} =================', fg="bright_cyan") files_validation_result.add(self.run_validation_on_package(path, error_ignore_list)) return all(files_validation_result)
def __init__(self, pack_path: str, update_type: Union[str, None], modified_files_in_pack: set, added_files: set, specific_version: str = None, pre_release: bool = False, pack: str = None, pack_metadata_only: bool = False, text: str = '', existing_rn_version_path: str = ''): self.pack = pack if pack else get_pack_name(pack_path) self.update_type = update_type self.pack_meta_file = PACKS_PACK_META_FILE_NAME try: self.pack_path = pack_name_to_path(self.pack) except TypeError: click.secho(f'Please verify the pack path is correct: {self.pack}.', fg='red') sys.exit(1) # renamed files will appear in the modified list as a tuple: (old path, new path) modified_files_in_pack = {file_[1] if isinstance(file_, tuple) else file_ for file_ in modified_files_in_pack} self.modified_files_in_pack = set() for file_path in modified_files_in_pack: self.modified_files_in_pack.add(self.check_for_release_notes_valid_file_path(file_path)) self.added_files = added_files self.pre_release = pre_release self.specific_version = specific_version self.existing_rn_changed = False self.text = text self.existing_rn_version_path = existing_rn_version_path self.should_delete_existing_rn = False self.pack_metadata_only = pack_metadata_only self.metadata_path = os.path.join(self.pack_path, 'pack_metadata.json') self.master_version = self.get_master_version()
def __init__(self, input: str, output: str = '', force: bool = False, marketplace: Optional[str] = None): """ Init a GenericModuleUnifier Args: input: a path of the GenericModule file to unify. output: The output dir to write the unified GenericModule json to. force: if True - Forcefully overwrites the preexisting unified GenericModule file if one exists. """ self.input_path = input self.pack_name = get_pack_name(file_path=self.input_path) self.pack_path = os.path.join(PACKS_DIR, self.pack_name) self.input_file_name = os.path.basename(self.input_path).rstrip('.json') self.use_force = force self.marketplace = marketplace if marketplace: MARKETPLACE_TAG_PARSER.marketplace = marketplace if output: if not os.path.isdir(output): click.secho('You have failed to provide a legal dir path', fg='bright_red') sys.exit(1) self.dest_dir = output else: # an output wasn't given, save the unified file in the input's file dir self.dest_dir = os.path.dirname(self.input_path) self.dest_path = os.path.join(self.dest_dir, f'{self.input_file_name}_unified.json')
def get_classifier_data(path): data = OrderedDict() json_data = get_json(path) id_ = json_data.get('id') name = json_data.get('name', '') fromversion = json_data.get('fromVersion') toversion = json_data.get('toVersion') pack = get_pack_name(path) incidents_types = set() default_incident_type = json_data.get('defaultIncidentType') if default_incident_type and default_incident_type != '': incidents_types.add(default_incident_type) key_type_map = json_data.get('keyTypeMap', {}) for key, value in key_type_map.items(): incidents_types.add(value) if name: data['name'] = name data['file_path'] = path if toversion: data['toversion'] = toversion if fromversion: data['fromversion'] = fromversion if pack: data['pack'] = pack if incidents_types: data['incident_types'] = list(incidents_types) return {id_: data}
def update_pack_releasenotes(**kwargs): _pack = kwargs.get('pack') update_type = kwargs.get('update_type') pre_release = kwargs.get('pre_release') is_all = kwargs.get('all') modified, added, old, _packs = FilesValidator( use_git=True).get_modified_and_added_files() packs_existing_rn = set() for pf in added: if 'ReleaseNotes' in pf: pack_with_existing_rn = get_pack_name(pf) packs_existing_rn.add(pack_with_existing_rn) if len(packs_existing_rn): existing_rns = ''.join(f"{p}, " for p in packs_existing_rn) print_warning( f"Found existing release notes for the following packs: {existing_rns.rstrip(', ')}" ) if len(_packs) > 1: pack_list = ''.join(f"{p}, " for p in _packs) if not is_all: if _pack: pass else: print_error( f"Detected changes in the following packs: {pack_list.rstrip(', ')}\n" f"To update release notes in a specific pack, please use the -p parameter " f"along with the pack name.") sys.exit(0) if len(modified) < 1: print_warning('No changes were detected.') sys.exit(0) if is_all and not _pack: packs = list(_packs - packs_existing_rn) packs_list = ''.join(f"{p}, " for p in packs) print_warning( f"Adding release notes to the following packs: {packs_list.rstrip(', ')}" ) for pack in packs: update_pack_rn = UpdateRN(pack=pack, update_type=update_type, pack_files=modified, pre_release=pre_release) update_pack_rn.execute_update() elif is_all and _pack: print_error( "Please remove the --all flag when specifying only one pack.") sys.exit(0) else: if _pack: if _pack in packs_existing_rn: print_error( f"New release notes file already found for {_pack}. " f"Please update manually or delete {pack_name_to_path(_pack)}" ) else: update_pack_rn = UpdateRN(pack=_pack, update_type=update_type, pack_files=modified, pre_release=pre_release) update_pack_rn.execute_update()
def get_incident_type_data(path): data = OrderedDict() json_data = get_json(path) id_ = json_data.get('id') name = json_data.get('name', '') fromversion = json_data.get('fromVersion') toversion = json_data.get('toVersion') playbook_id = json_data.get('playbookId') pre_processing_script = json_data.get('preProcessingScript') pack = get_pack_name(path) if name: data['name'] = name data['file_path'] = path if toversion: data['toversion'] = toversion if fromversion: data['fromversion'] = fromversion if pack: data['pack'] = pack if playbook_id and playbook_id != '': data['playbooks'] = playbook_id if pre_processing_script and pre_processing_script != '': data['scripts'] = pre_processing_script return {id_: data}
def get_layout_data(path): data = OrderedDict() json_data = get_json(path) layout = json_data.get('layout') name = layout.get('name', '-') id_ = json_data.get('id', layout.get('id', '-')) type_ = json_data.get('typeId') type_name = json_data.get('TypeName') fromversion = json_data.get('fromVersion') toversion = json_data.get('toVersion') kind = json_data.get('kind') pack = get_pack_name(path) if type_: data['typeID'] = type_ if type_name: data['typename'] = type_name data['name'] = name if toversion: data['toversion'] = toversion if fromversion: data['fromversion'] = fromversion if pack: data['pack'] = pack if kind: data['kind'] = kind data['path'] = path return {id_: data}
def is_valid_file(self, validate_rn=True, is_new_file=False, use_git=False, is_added_file=False): """Check whether the Incident Field is valid or not """ answers = [ super().is_valid_file(validate_rn), self.is_valid_type(), self.is_valid_group(), self.is_valid_content_flag(), self.is_valid_system_flag(), self.is_valid_cli_name(), self.is_valid_version(), self.is_valid_required(), self.does_not_have_empty_select_values(), self.is_aliased_fields_are_valid(), ] core_packs_list = get_core_pack_list() pack = get_pack_name(self.file_path) is_core = pack in core_packs_list if is_core: answers.append(self.is_valid_name()) if is_new_file and use_git: answers.append(self.is_valid_field_name_prefix()) if is_added_file: answers.append(self.is_valid_unsearchable_key()) return all(answers)
def are_release_notes_complete(self): is_valid = True modified_added_files = itertools.chain.from_iterable( (self.added_files or [], self.modified_files or [])) if modified_added_files: for file in modified_added_files: # renamed files will appear in the modified list as a tuple: (old path, new path) if isinstance(file, tuple): file = file[1] checked_file_pack_name = get_pack_name(file) if find_type(file) in SKIP_RELEASE_NOTES_FOR_TYPES: continue elif checked_file_pack_name and checked_file_pack_name == self.pack_name: # Refer image and description file paths to the corresponding yml files file = UpdateRN.change_image_or_desc_file_path(file) update_rn_util = UpdateRN(pack_path=self.pack_path, modified_files_in_pack=set(), update_type=None, added_files=set(), pack=self.pack_name) file_name, file_type = update_rn_util.get_changed_file_name_and_type( file) if file_name and file_type: if (RN_HEADER_BY_FILE_TYPE[file_type] not in self.latest_release_notes) or \ (file_name not in self.latest_release_notes): entity_name = update_rn_util.get_display_name(file) error_message, error_code = Errors.missing_release_notes_entry( file_type, self.pack_name, entity_name) if self.handle_error(error_message, error_code, self.release_notes_file_path): is_valid = False return is_valid
def create_output_dirs(self): # no output given if not self.dashboard_dir: # check if in content repository if not is_external_repository() and self.autocreate_dir: pack_name = get_pack_name(self.input) if self.logging: click.echo( f"No output path given creating Dashboards and GenericModules " f"directories in pack {pack_name}") pack_path = os.path.join(PACKS_DIR, pack_name) self.dashboard_dir = os.path.join(pack_path, DASHBOARDS_DIR) self.module_dir = os.path.join(pack_path, GENERIC_MODULES_DIR) # create the dirs, dont fail if exist self.create_output_dir(self.dashboard_dir) self.create_output_dir(self.module_dir) # if not in content create the files locally else: if self.logging: click.echo( "No output path given and not running in content repo, creating " "files in the current working directory") self.dashboard_dir = '.' self.module_dir = '.'
def validate_release_notes(self, file_path, added_files, modified_files, pack_error_ignore_list, is_modified): pack_name = get_pack_name(file_path) # modified existing RN if is_modified: error_message, error_code = Errors.modified_existing_release_notes(pack_name) if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path): return False # added new RN to a new pack if pack_name in self.new_packs: error_message, error_code = Errors.added_release_notes_for_new_pack(pack_name) if self.handle_error(error_message=error_message, error_code=error_code, file_path=file_path): return False if pack_name != 'NonSupported': if not added_files: added_files = {file_path} release_notes_validator = ReleaseNotesValidator(file_path, pack_name=pack_name, modified_files=modified_files, added_files=added_files, ignored_errors=pack_error_ignore_list, print_as_warnings=self.print_ignored_errors) return release_notes_validator.is_file_valid() return True
def __init__(self, file_path: str, ignored_errors=None, print_as_warnings=False, suppress_print=False, json_file_path=None, maximum_image_size: Optional[int] = None, specific_validations=None): super().__init__(file_path=file_path, ignored_errors=ignored_errors, print_as_warnings=print_as_warnings, suppress_print=suppress_print, json_file_path=json_file_path, specific_validations=specific_validations) self.pack_path = os.path.join(PACKS_DIR, get_pack_name(file_path)) self.maximum_image_size = maximum_image_size if maximum_image_size else self.IMAGE_MAX_SIZE
def get_playbook_data(file_path): playbook_data = OrderedDict() data_dictionary = get_yaml(file_path) id_ = data_dictionary.get('id', '-') name = data_dictionary.get('name', '-') deprecated = data_dictionary.get('deprecated', False) tests = data_dictionary.get('tests') toversion = data_dictionary.get('toversion') fromversion = data_dictionary.get('fromversion') implementing_scripts = get_task_ids_from_playbook('scriptName', data_dictionary) implementing_playbooks = get_task_ids_from_playbook('playbookName', data_dictionary) command_to_integration = get_commmands_from_playbook(data_dictionary) pack = get_pack_name(file_path) playbook_data['name'] = name playbook_data['file_path'] = file_path if toversion: playbook_data['toversion'] = toversion if fromversion: playbook_data['fromversion'] = fromversion if implementing_scripts: playbook_data['implementing_scripts'] = implementing_scripts if implementing_playbooks: playbook_data['implementing_playbooks'] = implementing_playbooks if command_to_integration: playbook_data['command_to_integration'] = command_to_integration if tests: playbook_data['tests'] = tests if deprecated: playbook_data['deprecated'] = deprecated if pack: playbook_data['pack'] = pack return {id_: playbook_data}
def get_layoutscontainer_data(path): json_data = get_json(path) layouts_container_fields = ["group", "edit", "indicatorsDetails", "indicatorsQuickView", "quickView", "close", "details", "detailsV2", "mobile", "name"] data = OrderedDict({field: json_data[field] for field in layouts_container_fields if json_data.get(field)}) id_ = json_data.get('id') pack = get_pack_name(path) incident_indicator_types_dependency = {id_} incident_indicator_fields_dependency = get_values_for_keys_recursively(json_data, ['fieldId']) if data.get('name'): incident_indicator_types_dependency.add(data['name']) if json_data.get('toVersion'): data['toversion'] = json_data['toVersion'] if json_data.get('fromVersion'): data['fromversion'] = json_data['fromVersion'] if pack: data['pack'] = pack data['file_path'] = path data['incident_and_indicator_types'] = list(incident_indicator_types_dependency) if incident_indicator_fields_dependency['fieldId']: data['incident_and_indicator_fields'] = incident_indicator_fields_dependency['fieldId'] return {id_: data}
def get_integration_data(file_path): integration_data = OrderedDict() data_dictionary = get_yaml(file_path) is_unified_integration = data_dictionary.get('script', {}).get('script', '') != '-' id_ = data_dictionary.get('commonfields', {}).get('id', '-') name = data_dictionary.get('name', '-') deprecated = data_dictionary.get('deprecated', False) tests = data_dictionary.get('tests') toversion = data_dictionary.get('toversion') fromversion = data_dictionary.get('fromversion') commands = data_dictionary.get('script', {}).get('commands', []) cmd_list = [command.get('name') for command in commands] pack = get_pack_name(file_path) integration_api_modules = get_integration_api_modules(file_path, data_dictionary, is_unified_integration) default_classifier = data_dictionary.get('defaultclassifier') default_incident_type = data_dictionary.get('defaultIncidentType') is_feed = data_dictionary.get('feed') mappers = set() deprecated_commands = [] for command in commands: if command.get('deprecated', False): deprecated_commands.append(command.get('name')) for mapper in ['defaultmapperin', 'defaultmapperout']: if data_dictionary.get(mapper): mappers.add(data_dictionary.get(mapper)) integration_data['name'] = name integration_data['file_path'] = file_path if toversion: integration_data['toversion'] = toversion if fromversion: integration_data['fromversion'] = fromversion if cmd_list: integration_data['commands'] = cmd_list if tests: integration_data['tests'] = tests if deprecated: integration_data['deprecated'] = deprecated if deprecated_commands: integration_data['deprecated_commands'] = deprecated_commands if pack: integration_data['pack'] = pack if integration_api_modules: integration_data['api_modules'] = integration_api_modules if default_classifier and default_classifier != '': integration_data['classifiers'] = default_classifier if mappers: integration_data['mappers'] = list(mappers) if default_incident_type and default_incident_type != '': integration_data['incident_types'] = default_incident_type if is_feed: integration_data['indicator_fields'] = "CommonTypes" integration_data['indicator_types'] = "CommonTypes" return {id_: integration_data}
def get_modified_files_for_testing( git_diff: str, ) -> Tuple[List[str], List[str], List[str], bool, List[str], set, bool, bool]: """ Gets git diff string and filters those files into tests: Args: git_diff: a git diff output (with --name-only flag) Returns: modified_files: Modified YMLs for testing (Integrations, Scripts, Playbooks). modified_tests: Test playbooks. changed_common_files: Globally used YMLs (Like CommonServerPython). is_conf_json: If Tests/Conf.json has been changed. sample_tests: Files to test, Like the infrastructures files. modified_metadata: Pack names of changed metadata files. is_reputations_json: If any reputation file changed. is_indicator_json: If any indicator file changed. """ types_to_files: Dict[FileType, Set[str]] = create_type_to_file( git_diff) # Mapping of the files FileType: file path # Checks if any common file exists in types_to_file changed_common_files = get_common_files( types_to_files.get(FileType.SCRIPT, set())) types_to_files = remove_common_files(types_to_files, changed_common_files) # Sample tests are the remaining python files sample_tests = types_to_files.get(FileType.PYTHON_FILE, set()) # Modified files = YMLs of integrations, scripts and playbooks modified_files: Set[str] = types_to_files.get( FileType.INTEGRATION, set()).union( types_to_files.get(FileType.SCRIPT, set()), types_to_files.get(FileType.PLAYBOOK, set()) ) # Modified YMLs for testing (Integrations, Scripts, Playbooks). # Metadata packs modified_metadata: Set[str] = set() for file_path in types_to_files.get(FileType.METADATA, set()): modified_metadata.add(tools.get_pack_name(file_path)) modified_tests: Set[str] = types_to_files.get( FileType.TEST_PLAYBOOK, set()) # Modified tests are test playbooks # Booleans. If this kind of file is inside, its exists is_conf_json = FileType.CONF_JSON in types_to_files is_reputations_json = FileType.REPUTATION in types_to_files is_indicator_json = FileType.INDICATOR_FIELD in types_to_files return ( list(modified_files), list(modified_tests), list(changed_common_files), is_conf_json, list(sample_tests), modified_metadata, is_reputations_json, is_indicator_json, )
def update_checked_flags_by_support_level(self, file_path): pack_name = get_pack_name(file_path) if pack_name: metadata_path = os.path.join(PACKS_DIR, pack_name, PACKS_PACK_META_FILE_NAME) metadata_json = self.get_metadata_file_content(metadata_path) support = metadata_json.get(PACK_METADATA_SUPPORT) if support in ('partner', 'community'): self.add_flag_to_ignore_list(file_path, support)
def get_packs(changed_files): packs = set() for changed_file in changed_files: if isinstance(changed_file, tuple): changed_file = changed_file[1] pack = get_pack_name(changed_file) if pack: packs.add(pack) return packs
def check_existing_rn(self, added_files: set): """ Checks whether the packs already have an existing release notes files and adds them to the packs_existing_rn dictionary. :param added_files: A set of new added files """ for file_path in added_files: if 'ReleaseNotes' in file_path: self.packs_existing_rn[get_pack_name(file_path)] = file_path
def get_modified_files_for_testing(git_diff: str) -> ModifiedFiles: """ Gets git diff string and filters those files into tests: Args: git_diff: a git diff output (with --name-only flag) Returns: ModifiedFiles instance """ types_to_files: Dict[FileType, Set[str]] = create_type_to_file( git_diff) # Mapping of the files FileType: file path # Checks if any common file exists in types_to_file changed_common_files = get_common_files( types_to_files.get(FileType.SCRIPT, set())) types_to_files = remove_common_files(types_to_files, changed_common_files) # Sample tests are the remaining python files sample_tests = types_to_files.get(FileType.PYTHON_FILE, set()) # Modified files = YMLs of integrations, scripts and playbooks modified_files: Set[str] = types_to_files.get( FileType.INTEGRATION, set()).union( types_to_files.get(FileType.SCRIPT, set()), types_to_files.get(FileType.BETA_INTEGRATION, set()), types_to_files.get(FileType.PLAYBOOK, set()) ) # Modified YMLs for testing (Integrations, Scripts, Playbooks). # Metadata packs modified_metadata: Set[str] = set() for file_path in types_to_files.get(FileType.METADATA, set()): modified_metadata.add(tools.get_pack_name(file_path)) modified_tests: Set[str] = types_to_files.get( FileType.TEST_PLAYBOOK, set()) # Modified tests are test playbooks # Booleans. If this kind of file is inside, its exists is_conf_json = FileType.CONF_JSON in types_to_files is_landing_page_sections_json = FileType.LANDING_PAGE_SECTIONS_JSON in types_to_files is_reputations_json = FileType.REPUTATION in types_to_files is_indicator_json = FileType.INDICATOR_FIELD in types_to_files modified_files_instance = ModifiedFiles(list(modified_files), list(modified_tests), list(changed_common_files), is_conf_json, list(sample_tests), modified_metadata, is_reputations_json, is_indicator_json, is_landing_page_sections_json) return modified_files_instance
def ignore_secrets(self, secrets): pack_dir = get_pack_name(self.full_output_path) try: with open(f'Packs/{pack_dir}/.secrets-ignore', 'a') as f: for secret in secrets: f.write(secret) f.write('\n') except FileNotFoundError: print_warning( "Could not find the .secrets-ignore file - make sure your path is correct" )
def validate_added_files(self, added_files, modified_files): click.secho(f'\n================= Running validation on newly added files =================', fg="bright_cyan") valid_files = set() for file_path in added_files: pack_name = get_pack_name(file_path) valid_files.add(self.run_validations_on_file(file_path, self.get_error_ignore_list(pack_name), is_modified=False, modified_files=modified_files, added_files=added_files)) return all(valid_files)
def get_packs(modified_files, added_files): packs = set() changed_files = modified_files.union(added_files) for changed_file in changed_files: if isinstance(changed_file, tuple): changed_file = changed_file[1] pack = get_pack_name(changed_file) if pack and is_file_path_in_pack(changed_file): packs.add(pack) return packs
def get_playbook_data(file_path: str) -> dict: playbook_data = OrderedDict() data_dictionary = get_yaml(file_path) graph = build_tasks_graph(data_dictionary) id_ = data_dictionary.get('id', '-') name = data_dictionary.get('name', '-') deprecated = data_dictionary.get('deprecated', False) tests = data_dictionary.get('tests') toversion = data_dictionary.get('toversion') fromversion = data_dictionary.get('fromversion') implementing_scripts, implementing_scripts_skippable = get_task_ids_from_playbook('scriptName', data_dictionary, graph ) implementing_playbooks, implementing_playbooks_skippable = get_task_ids_from_playbook('playbookName', data_dictionary, graph ) command_to_integration, command_to_integration_skippable = get_commands_from_playbook(data_dictionary) skippable_tasks = (implementing_scripts_skippable + implementing_playbooks_skippable + command_to_integration_skippable) pack = get_pack_name(file_path) dependent_incident_fields, dependent_indicator_fields = get_dependent_incident_and_indicator_fields(data_dictionary) playbook_data['name'] = name playbook_data['file_path'] = file_path if toversion: playbook_data['toversion'] = toversion if fromversion: playbook_data['fromversion'] = fromversion if implementing_scripts: playbook_data['implementing_scripts'] = implementing_scripts if implementing_playbooks: playbook_data['implementing_playbooks'] = implementing_playbooks if command_to_integration: playbook_data['command_to_integration'] = command_to_integration if tests: playbook_data['tests'] = tests if deprecated: playbook_data['deprecated'] = deprecated if pack: playbook_data['pack'] = pack if skippable_tasks: playbook_data['skippable_tasks'] = skippable_tasks if dependent_incident_fields: playbook_data['incident_fields'] = list(dependent_incident_fields) if dependent_indicator_fields: playbook_data['indicator_fields'] = list(dependent_indicator_fields) return {id_: playbook_data}
def has_unskipped_test_playbook(self, current_file, entity_id, file_path, test_playbook_ids=None): """Check if the content entity has at least one unskipped test playbook. Collect test playbook ids from the `tests` field in the file, merge them with provided test_playbook_ids and validate at least one is unskipped. Args: current_file: The file to check. entity_id: The id of the entity to check. file_path: The file path of the entity to check. test_playbook_ids: test_playbook_ids unrelated to `tests` field in the file. Returns: True if the content entity has at least one unskipped test playbook. """ # If it has a dynamic section tag, it shouldn't have a test playbook. if self.DYNAMIC_SECTION_TAG in current_file.get('tags', []): return True if test_playbook_ids is None: test_playbook_ids = [] test_playbooks_unskip_status = {} all_test_playbook_ids = test_playbook_ids.copy() skipped_tests = self.conf_data.get('skipped_tests', {}) # do not check this validation for ApiModules pack if get_pack_name(file_path) == API_MODULES_PACK: return self._is_valid if isinstance(current_file.get('tests'), list): all_test_playbook_ids.extend(current_file.get('tests', [])) for test_playbook_id in set(all_test_playbook_ids): if (skipped_tests and test_playbook_id in skipped_tests) or 'No test' in test_playbook_id: test_playbooks_unskip_status[test_playbook_id] = False else: test_playbooks_unskip_status[test_playbook_id] = True if not any(test_playbooks_unskip_status.values() ) and not self.has_unittest(file_path): error_message, error_code = Errors.all_entity_test_playbooks_are_skipped( entity_id) if self.handle_error(error_message, error_code, file_path=file_path): self._is_valid = False return self._is_valid
def update_checked_flags_by_support_level(self, file_path): pack_name = get_pack_name(file_path) if pack_name: metadata_path = os.path.join(PACKS_DIR, pack_name, PACKS_PACK_META_FILE_NAME) metadata_json = self.get_metadata_file_content(metadata_path) support = metadata_json.get(PACK_METADATA_SUPPORT) certification = metadata_json.get(PACK_METADATA_CERTIFICATION) if support == 'partner': if certification is not None and certification != 'certified': self.add_flag_to_ignore_list(file_path, 'non-certified-partner') elif support == 'community': self.add_flag_to_ignore_list(file_path, 'community')
def get_incident_field_data(path, incidents_types_list): data = OrderedDict() json_data = get_json(path) id_ = json_data.get('id') name = json_data.get('name', '') fromversion = json_data.get('fromVersion') toversion = json_data.get('toVersion') pack = get_pack_name(path) all_associated_types = set() all_scripts = set() associated_types = json_data.get('associatedTypes') if associated_types: all_associated_types = set(associated_types) system_associated_types = json_data.get('systemAssociatedTypes') if system_associated_types: all_associated_types = all_associated_types.union( set(system_associated_types)) if 'all' in all_associated_types: all_associated_types = [ list(incident_type.keys())[0] for incident_type in incidents_types_list ] scripts = json_data.get('script') if scripts: all_scripts = {scripts} field_calculations_scripts = json_data.get('fieldCalcScript') if field_calculations_scripts: all_scripts = all_scripts.union({field_calculations_scripts}) if name: data['name'] = name data['file_path'] = path if toversion: data['toversion'] = toversion if fromversion: data['fromversion'] = fromversion if pack: data['pack'] = pack if all_associated_types: data['incident_types'] = list(all_associated_types) if all_scripts: data['scripts'] = list(all_scripts) return {id_: data}