def init(path): """determines size and number of files""" log_comment('initializing...') total_size = 0 count = 0 for root, dirs, files in os.walk(path): for skip in SKIP: if skip in dirs: dirs.remove(skip) for filename in files: apath = os.path.join(root, filename) count += 1 try: total_size += os.path.getsize(apath) except OSError as e: log_error(str(e), filename) readable = total_size for unit in ['bytes', 'KiB', 'MiB', 'GiB', 'TiB']: if readable < 1024: log_comment('%d files, %3.1f %s' % (count, readable, unit)) return count readable /= 1024.0
def blueflower(path, hashesfile, pwd, output_file): """runs blueflower, returns name of the log file""" global RGX_INFILE global RGX_INFILENAME if not os.path.exists(path): raise BFException('%s does not exist' % path) if hashesfile and not os.path.exists(hashesfile): raise BFException('%s does not exist' % hashesfile) if output_file: if os.path.basename(output_file): logfile = output_file else: logfile = output_file + '/%s-%s.csv' % (PROGRAM, timestamp()) else: logfile = '%s-%s.csv' % (PROGRAM, timestamp()) # reset any existing logger logger = logging.getLogger() if logger.handlers: logger.handlers[0].stream.close() logger.removeHandler(logger.handlers[0]) # instantiate logger logging.basicConfig(filename=logfile, format='%(message)s', level=logging.INFO) banner() log_comment('writing to %s' % logfile) # hash file support if hashesfile and pwd: try: get_hashes(hashesfile, pwd) except BFException: raise # precompile the regexes rgx_infile = '|'.join(INFILE) try: RGX_INFILE = re.compile(rgx_infile, re.IGNORECASE) except re.error: raise BFException('invalid infile regex %s' % rgx_infile) rgx_infilename = '|'.join(INFILENAME) try: RGX_INFILENAME = re.compile(rgx_infilename, re.IGNORECASE) except re.error: raise BFException('invalid infilename regex %s' % rgx_infilename) # start slow operations count = init(path) scan(path, count) count_logged(logfile) return logfile
def scan(path, count): """selects files to process, checks file names""" log_comment('scanning %s:' % path) scanned = 0 bar_width = 32 if count < bar_width: bar_width = count if count == 0: bar_width = 1 sys.stdout.write('%s\n' % ("=" * bar_width)) bar_blocksize = count / bar_width bar_left = bar_width bar_count = 0 for root, dirs, files in os.walk(path): for skip in SKIP: if skip in dirs: dirs.remove(skip) for filename in files: abspath = os.path.abspath(os.path.join(root, filename)) res = RGX_INFILENAME.search(filename.lower()) if res: log_secret(res.group(), abspath) try: ftype, supported = type_file(abspath) except TypeError as e: log_error(str(e), abspath) continue if supported: if ftype in ENCRYPTED: # report but do not process log_encrypted(ftype, abspath) if ftype in EXE: # report but do not process if looks_uniform(filename=abspath): log_packed(ftype, abspath) else: log_exe(ftype, abspath) else: # process the file do_file(ftype, abspath) scanned += 1 # update progress bar bar_count += 1 if bar_count >= bar_blocksize and bar_left: sys.stdout.write("=") sys.stdout.flush() bar_count = 0 bar_left -= 1 sys.stdout.write("\n") log_comment('%d files supported were processed' % scanned) return scanned
def load_dictionary_file(afile): log_comment('adding custom dictionary %s to infile' % afile) try: fid = open(afile) except IOError as e: log_error(str(e), afile) return data = fid.read().lower() fid.close() return data.splitlines()
def count_logged(logfile): logs = open(logfile).read() secrets = logs.count('SECRET,') log_comment('%d files or strings may contain secrets' % secrets) encrypted = logs.count('ENCRYPTED,') log_comment('%d files look encrypted' % encrypted) exe = logs.count('EXE,') packed = logs.count('EXE PACKED,') log_comment('%d files look executable (including %d packed)' % \ (exe+packed, packed))
def get_hashes(hashesfile, pwd): """create hashes list from hashesfile using the given password""" global HASHES global HASH_KEY global HASH_REGEX log_comment('verifying hashes file %s...' % hashesfile) fin = open(hashesfile) regex = fin.readline().rstrip('\n') try: (salt, verifier_from_file) = fin.readline().rstrip('\n').split(',') except ValueError: raise BFException('failed to extract verifier and salt') (key, verifier_from_pwd, salt) = key_derivation(pwd, salt) fail = False if verifier_from_pwd != verifier_from_file: log_comment('verifier does not match (incorrect password?)') fail = True else: HASH_KEY = key HASH_REGEX = regex try: re.compile(regex) except re.error: log_comment('invalid regex') fail = True # file pointer is now at the 3rd line: hashes = [] for line in fin: ahash = line.strip() if len(ahash) != 2 * HASH_BYTES: log_comment('invalid hash length (%d bytes): %s' % (len(ahash), ahash)) fail = True # check that the hash is an hex value try: int(ahash, 16) except ValueError: log_comment('invalid hash value: %s' % ahash) fail = True if not fail: hashes.append(ahash) if fail: raise BFException('hashes file failed to verify') # record hashes and key, notifies of duplicates HASHES = frozenset(hashes) log_comment('%d hashes read, %d uniques' % (len(hashes), len(HASHES))) log_comment('using regex %s' % HASH_REGEX) log_comment('hashes file successfully verified')
def signal_handler(*_): """interrupt upon ^C""" sys.stdout.write("\n") log_comment('SIGINT received, quitting') sys.exit(0)
def blueflower(path, hashesfile, dictionaryfile, pwd, output_file, exclude_default_dictionary): """runs blueflower, returns name of the log file""" global RGX_INFILE global RGX_INFILENAME if not os.path.exists(path): raise BFException('%s does not exist' % path) if hashesfile and not os.path.exists(hashesfile): raise BFException('%s does not exist' % hashesfile) if dictionaryfile and not os.path.exists(dictionaryfile): raise BFException('%s does not exist' % dictionaryfile) if output_file: if os.path.isfile(output_file): logfile = output_file else: logfile = output_file + '/%s-%s-%s.csv' % (PROGRAM, os.path.basename(os.path.normpath(path)), timestamp()) else: logfile = '%s-%s-%s.csv' % (PROGRAM, os.path.basename(os.path.normpath(path)), timestamp()) # reset any existing logger logger = logging.getLogger() if logger.handlers: logger.handlers[0].stream.close() logger.removeHandler(logger.handlers[0]) # instantiate logger logging.basicConfig(filename=logfile, format='%(message)s', level=logging.INFO) banner() log_comment('writing to %s' % logfile) # hash file support if hashesfile and pwd: try: get_hashes(hashesfile, pwd) except BFException: raise # read the dictionary and add to INFILE if dictionaryfile: extradictionary = load_dictionary_file(dictionaryfile) else: extradictionary=[] for rex in extradictionary: try: re.compile(rex, re.IGNORECASE) except re.error: log_error('regex does not compile: %s' % rex) # configure the regex dictionary to be used if exclude_default_dictionary: rgx_infile = '|'.join(set(extradictionary)) else: rgx_infile = '|'.join(set(INFILE) | set(extradictionary)) log_comment(rgx_infile) # precompile the regexes try: RGX_INFILE = re.compile(rgx_infile, re.IGNORECASE) except re.error: raise BFException('invalid infile regex %s' % rgx_infile) rgx_infilename = '|'.join(INFILENAME) try: RGX_INFILENAME = re.compile(rgx_infilename, re.IGNORECASE) except re.error: raise BFException('invalid infilename regex %s' % rgx_infilename) # start slow operations count = init(path) scan(path, count) count_logged(logfile) return logfile
def get_hashes(hashesfile, pwd): """create hashes list from hashesfile using the given password""" global HASHES global HASH_REGEX log_comment('verifying hashes file %s...' % hashesfile) fin = open(hashesfile) regex = fin.readline().rstrip('\n') try: (salt, verifier_from_file) = fin.readline().rstrip('\n').split(',') except ValueError: raise BFException('failed to extract verifier and salt') (key, verifier_from_pwd, salt) = key_derivation(pwd, salt) if verifier_from_pwd != verifier_from_file: log_comment('verifier does not match (incorrect password?)') fail() else: HASH_REGEX = regex try: re.compile(regex) except re.error: log_comment('invalid regex') fail() # file pointer is now at the 3rd line: hashes = [] for line in fin: hash_line = line.strip().split(',') ahash = hash_line[0] if len(hash_line) != 2: log_comment('Missing salt for this secret: %s' % ahash) fail() ahash_salt = hash_line[1] ahash_key, _, _ = key_derivation(pwd, ahash_salt) if len(ahash) != 2*HASH_BYTES: log_comment('invalid hash length (%d bytes): %s' % (len(ahash), ahash)) fail() # check that the hash is an hex value try: int(ahash, 16) except ValueError: log_comment('invalid hash value: %s' % ahash) fail() hashes.append((ahash, ahash_key)) # record hashes and key, notifies of duplicates HASHES = frozenset(hashes) log_comment('%d hashes read, %d uniques' % (len(hashes), len(HASHES))) log_comment('using regex %s' % HASH_REGEX) log_comment('hashes file successfully verified')