def create_file_release_notes(change_type, full_file_name): """ Create release note for changed file. :param change_type: git change status (A, M, R*) :param full_file_name: path to file in repository :return: None """ if isinstance(full_file_name, tuple): _, full_file_name = full_file_name is_pack = is_file_path_in_pack(full_file_name) if is_pack: file_type = full_file_name.split("/")[2] else: file_type = full_file_name.split("/")[0] base_name = os.path.basename(full_file_name) file_suffix = os.path.splitext(base_name)[-1] file_type_mapping = RELEASE_NOTE_GENERATOR.get(file_type) if file_type_mapping is None or file_suffix not in CONTENT_FILE_SUFFIXES: print_warning("Unsupported file type: {}".format(full_file_name)) return if change_type != "R100": # only file name has changed (no actual data was modified if 'R' in change_type: # handle the same as modified change_type = 'M' file_type_mapping.add(change_type, CONTENT_LIB_PATH + full_file_name)
def get_packs(modified_files, added_files): packs = set() changed_files = modified_files.union(added_files) for changed_file in changed_files: if isinstance(changed_file, tuple): changed_file = changed_file[1] pack = get_pack_name(changed_file) if pack and is_file_path_in_pack(changed_file): packs.add(pack) return packs
def search_potential_secrets(secrets_file_paths: list): """Returns potential secrets(sensitive data) found in committed and added files :param secrets_file_paths: paths of files that are being commited to git repo :return: dictionary(filename: (list)secrets) of strings sorted by file name for secrets found in files """ secrets_found = {} for file_path in secrets_file_paths: # Get if file path in pack and pack name is_pack = is_file_path_in_pack(file_path) pack_name = get_pack_name(file_path) # Get generic/ioc/files white list sets based on if pack or not secrets_white_list, ioc_white_list, files_white_list = get_white_listed_items( is_pack, pack_name) # Skip white listed files if file_path in files_white_list: print( "Skipping secrets detection for file: {} as it is white listed" .format(file_path)) continue # Init vars for current loop file_name = os.path.basename(file_path) high_entropy_strings = [] secrets_found_with_regex = [] _, file_extension = os.path.splitext(file_path) skip_secrets = {'skip_once': False, 'skip_multi': False} # get file contents file_contents = get_file_contents(file_path, file_extension) # in packs regard all items as regex as well, reset pack's whitelist in order to avoid repetition later if is_pack: file_contents = remove_white_list_regex(file_contents, secrets_white_list) secrets_white_list = set() yml_file_contents = get_related_yml_contents(file_path) # Add all context output paths keywords to whitelist temporary if file_extension == YML_FILE_EXTENSION or yml_file_contents: temp_white_list = create_temp_white_list( yml_file_contents if yml_file_contents else file_contents) secrets_white_list = secrets_white_list.union(temp_white_list) # Search by lines after strings with high entropy / IoCs regex as possibly suspicious for line in file_contents.split('\n'): # if detected disable-secrets comments, skip the line/s skip_secrets = is_secrets_disabled(line, skip_secrets) if skip_secrets['skip_once'] or skip_secrets['skip_multi']: skip_secrets['skip_once'] = False continue # REGEX scanning for IOCs and false positive groups regex_secrets, false_positives = regex_for_secrets(line) for regex_secret in regex_secrets: if not any(ioc.lower() in regex_secret.lower() for ioc in ioc_white_list): secrets_found_with_regex.append(regex_secret) # added false positives into white list array before testing the strings in line secrets_white_list = secrets_white_list.union(false_positives) # due to nature of eml files, skip string by string secret detection - only regex if file_extension in SKIP_FILE_TYPE_ENTROPY_CHECKS or \ any(demisto_type in file_name for demisto_type in SKIP_DEMISTO_TYPE_ENTROPY_CHECKS): continue line = remove_false_positives(line) # calculate entropy for each string in the file for string_ in line.split(): # compare the lower case of the string against both generic whitelist & temp white list if not any(white_list_string.lower() in string_.lower() for white_list_string in secrets_white_list): entropy = calculate_shannon_entropy(string_) if entropy >= ENTROPY_THRESHOLD: high_entropy_strings.append(string_) if high_entropy_strings or secrets_found_with_regex: # uniquify identical matches between lists file_secrets = list( set(high_entropy_strings + secrets_found_with_regex)) secrets_found[file_name] = file_secrets return secrets_found