def is_secrets_disabled(line, skip_secrets): if bool(re.findall(r'(disable-secrets-detection-start)', line)): skip_secrets['skip_multi'] = True elif bool(re.findall(r'(disable-secrets-detection-end)', line)): skip_secrets['skip_multi'] = False elif bool(re.findall(r'(disable-secrets-detection)', line)): skip_secrets['skip_once'] = True return skip_secrets
def ignore_base64(file_contents): base64_strings = re.findall( r'(?:[A-Za-z0-9+/]{4})*(?:[A-Za-z0-9+/]{2}==|' r'[A-Za-z0-9+/]{3}=|[A-Za-z0-9+/]{4})', file_contents) for base64_string in base64_strings: if len(base64_string) > 500: file_contents = file_contents.replace(base64_string, '') return file_contents
def create_temp_white_list(file_contents): temp_white_list = set() context_paths = re.findall(r'contextPath: (\S+\.+\S+)', file_contents) for context_path in context_paths: context_path = context_path.split('.') context_path = [ white_item.lower() for white_item in context_path if len(white_item) > 4 ] temp_white_list = temp_white_list.union(context_path) return temp_white_list
def regex_for_secrets(line): """Scans for IOCs with potentially low entropy score :param line: line to test as string representation (string) :return potential_secrets (list) IOCs found via regex, false_positives (list) Non secrets with high entropy """ potential_secrets = [] false_positives = [] # Dates REGEX for false positive preventing since they have high entropy dates = re.findall(DATES_REGEX, line) if dates: false_positives += [date[0].lower() for date in dates] # UUID REGEX uuids = re.findall(UUID_REGEX, line) if uuids: false_positives += uuids # docker images version are detected as ips. so we ignore and whitelist them # example: dockerimage: demisto/duoadmin:1.0.0.147 re_res = re.search(r'dockerimage:\s*\w*demisto/\w+:(\d+.\d+.\d+.\d+)', line) if re_res: docker_version = re_res.group(1) false_positives.append(docker_version) line = line.replace(docker_version, '') # URL REGEX urls = re.findall(URLS_REGEX, line) if urls: potential_secrets += urls # EMAIL REGEX emails = re.findall(EMAIL_REGEX, line) if emails: potential_secrets += emails # IPV6 REGEX ipv6_list = re.findall(IPV6_REGEX, line) if ipv6_list: for ipv6 in ipv6_list: if ipv6 != '::' and len(ipv6) > 4: potential_secrets.append(ipv6) # IPV4 REGEX ipv4_list = re.findall(IPV4_REGEX, line) if ipv4_list: potential_secrets += ipv4_list return potential_secrets, false_positives