def run_plugins(self):
log.info("Selected plugins: " + str(self.selected_plugins))
completed_plugins = []
for plugin in self.selected_plugins:
# First check built-in plugins that ship with Hindsight
# log.info(" Built-in Plugins:")
for standard_plugin in pyhindsight.plugins.__all__:
# Check if the standard plugin is the selected_plugin we're looking for
if standard_plugin == plugin:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.info(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
continue
log.info(" - Loading '{}' [standard plugin]".format(plugin))
try:
module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
except ImportError, e:
log.error(" - Error: {}".format(e))
print format_plugin_output(plugin, "-unknown", 'import failed (see log)')
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(self)
print format_plugin_output(module.friendlyName, module.version, parsed_items)
self.plugin_results[plugin] = [module.friendlyName, module.version, parsed_items]
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
break
except Exception, e:
print format_plugin_output(module.friendlyName, module.version, 'failed')
self.plugin_results[plugin] = [module.friendlyName, module.version, 'failed']
log.info(" - Failed; {}".format(e))
def run_plugins(self):
log.info("Selected plugins: " + str(self.selected_plugins))
completed_plugins = []
for plugin in self.selected_plugins:
# First check built-in plugins that ship with Hindsight
# log.info(" Built-in Plugins:")
for standard_plugin in pyhindsight.plugins.__all__:
# Check if the standard plugin is the selected_plugin we're looking for
if standard_plugin == plugin:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.info(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
continue
log.info(" - Loading '{}' [standard plugin]".format(plugin))
try:
module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
except ImportError, e:
log.error(" - Error: {}".format(e))
print format_plugin_output(plugin, "-unknown", 'import failed (see log)')
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(self)
print format_plugin_output(module.friendlyName, module.version, parsed_items)
self.plugin_results[plugin] = [module.friendlyName, module.version, parsed_items]
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
break
except Exception, e:
print format_plugin_output(module.friendlyName, module.version, 'failed')
self.plugin_results[plugin] = [module.friendlyName, module.version, 'failed']
log.info(" - Failed; {}".format(e))
def main():
def write_excel(analysis_session):
import StringIO
# Set up a StringIO object to save the XLSX content to before saving to disk
string_buffer = StringIO.StringIO()
# Generate the XLSX content using the function in the AnalysisSession and save it to the StringIO object
analysis_session.generate_excel(string_buffer)
# Go back to the beginning (be kind, rewind)
string_buffer.seek(0)
# Write the StringIO object to a file on disk named what the user specified
with open(
"{}.{}".format(
os.path.join(real_path, analysis_session.output_name),
analysis_session.selected_output_format),
'wb') as file_output:
shutil.copyfileobj(string_buffer, file_output)
def write_sqlite(analysis_session):
output_file = analysis_session.output_name + '.sqlite'
if not os.path.exists(output_file):
analysis_session.generate_sqlite(output_file)
else:
print(
"\n Database file \"{}\" already exists. Please choose a different output location.\n"
.format(output_file))
print(banner)
# Useful when Hindsight is run from a different directory than where the file is located
real_path = os.path.dirname(os.path.realpath(sys.argv[0]))
# Set up the AnalysisSession object, and transfer the relevant input arguments to it
analysis_session = AnalysisSession()
# parse_arguments needs the analysis_session as an input to set things like available decrypts
args = parse_arguments(analysis_session)
if args.output:
analysis_session.output_name = args.output
if args.cache:
analysis_session.cache_path = args.cache
analysis_session.selected_output_format = args.format
analysis_session.browser_type = args.browser_type
analysis_session.timezone = args.timezone
if args.log == 'hindsight.log':
args.log = os.path.join(real_path, args.log)
analysis_session.log_path = args.log
# Set up logging
logging.basicConfig(
filename=analysis_session.log_path,
level=logging.DEBUG,
format='%(asctime)s.%(msecs).03d | %(levelname).01s | %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger(__name__)
# Hindsight version info
log.info(
'\n' + '#' * 80 +
'\n### Hindsight v{} (https://github.com/obsidianforensics/hindsight) ###\n'
.format(pyhindsight.__version__) + '#' * 80)
# Analysis start time
print(format_meta_output("Start time", str(datetime.datetime.now())[:-3]))
# Read the input directory
analysis_session.input_path = args.input
print(format_meta_output("Input directory", args.input))
log.info("Reading files from %s" % args.input)
input_listing = os.listdir(args.input)
log.debug("Input directory contents: " + str(input_listing))
# Search input directory for browser profiles to analyze
input_profiles = analysis_session.find_browser_profiles(args.input)
log.info(" - Found {} browser profile(s): {}".format(
len(input_profiles), input_profiles))
analysis_session.profile_paths = input_profiles
print(
format_meta_output(
"Output name",
"{}.{}".format(analysis_session.output_name,
analysis_session.selected_output_format)))
# Run the AnalysisSession
print("\n Processing:")
analysis_session.run()
print("\n Running plugins:")
log.info("Plugins:")
completed_plugins = []
# First run built-in plugins that ship with Hindsight
log.info(" Built-in Plugins:")
for plugin in pyhindsight.plugins.__all__:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
continue
log.debug(" - Loading '{}'".format(plugin))
try:
module = importlib.import_module(
"pyhindsight.plugins.{}".format(plugin))
except ImportError, e:
log.error(" - Error: {}".format(e))
print(
format_plugin_output(plugin, "-unknown",
'import failed (see log)'))
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(analysis_session)
print(
format_plugin_output(module.friendlyName, module.version,
parsed_items))
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception, e:
print(
format_plugin_output(module.friendlyName, module.version,
'failed'))
log.info(" - Failed; {}".format(e))
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.debug(
" - Skipping '{}'; a plugin with that name has run already"
.format(plugin))
continue
log.debug(" - Loading '{}'".format(plugin))
try:
module = __import__(plugin)
except ImportError, e:
log.error(" - Error: {}".format(e))
print(
format_plugin_output(
plugin, "-unknown",
'import failed (see log)'))
continue
try:
log.info(" - Running '{}' plugin".format(
module.friendlyName))
parsed_items = module.plugin(analysis_session)
print(
format_plugin_output(
module.friendlyName, module.version,
parsed_items))
log.info(
" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception, e:
print(
def main():
def write_excel(analysis_session):
import io
# Set up a StringIO object to save the XLSX content to before saving to disk
string_buffer = io.BytesIO()
# Generate the XLSX content using the function in the AnalysisSession and save it to the StringIO object
analysis_session.generate_excel(string_buffer)
# Go back to the beginning (be kind, rewind)
string_buffer.seek(0)
# Write the StringIO object to a file on disk named what the user specified
with open(f'{os.path.join(real_path, analysis_session.output_name)}.{analysis_session.selected_output_format}', 'wb') as file_output:
shutil.copyfileobj(string_buffer, file_output)
def write_sqlite(analysis_session):
output_file = analysis_session.output_name + '.sqlite'
if os.path.exists(output_file):
if os.path.getsize(output_file) > 0:
print(('\nDatabase file "{}" already exists.\n'.format(output_file)))
user_input = input('Would you like to (O)verwrite it, (R)ename output file, or (E)xit? ')
over_re = re.compile(r'(^o$|overwrite)', re.IGNORECASE)
rename_re = re.compile(r'(^r$|rename)', re.IGNORECASE)
exit_re = re.compile(r'(^e$|exit)', re.IGNORECASE)
if re.search(exit_re, user_input):
print("Exiting... ")
sys.exit()
elif re.search(over_re, user_input):
os.remove(output_file)
print(("Deleted old \"%s\"" % output_file))
elif re.search(rename_re, user_input):
output_file = "{}_1.sqlite".format(output_file[:-7])
print(("Renaming new output to {}".format(output_file)))
else:
print("Did not understand response. Exiting... ")
sys.exit()
analysis_session.generate_sqlite(output_file)
def write_jsonl(analysis_session):
output_file = analysis_session.output_name + '.jsonl'
analysis_session.generate_jsonl(output_file)
print(banner)
# Useful when Hindsight is run from a different directory than where the file is located
real_path = os.path.dirname(os.path.realpath(sys.argv[0]))
# Set up the AnalysisSession object, and transfer the relevant input arguments to it
analysis_session = AnalysisSession()
# parse_arguments needs the analysis_session as an input to set things like available decrypts
args = parse_arguments(analysis_session)
if args.output:
analysis_session.output_name = args.output
if args.cache:
analysis_session.cache_path = args.cache
analysis_session.selected_output_format = args.format
analysis_session.browser_type = args.browser_type
analysis_session.timezone = args.timezone
if args.log == 'hindsight.log':
args.log = os.path.join(real_path, args.log)
analysis_session.log_path = args.log
# Set up logging
logging.basicConfig(filename=analysis_session.log_path, level=logging.DEBUG,
format='%(asctime)s.%(msecs).03d | %(levelname).01s | %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger(__name__)
# Hindsight version info
log.info(
'\n' + '#' * 80 + '\n### Hindsight v{} (https://github.com/obsidianforensics/hindsight) ###\n'
.format(pyhindsight.__version__) + '#' * 80)
# Analysis start time
print((format_meta_output("Start time", str(datetime.datetime.now())[:-3])))
# Read the input directory
analysis_session.input_path = args.input
print((format_meta_output('Input directory', args.input)))
log.info(f'Reading files from {args.input}')
input_listing = os.listdir(args.input)
log.debug("Input directory contents: " + str(input_listing))
# Search input directory for browser profiles to analyze
input_profiles = analysis_session.find_browser_profiles(args.input)
log.info(f' - Found {len(input_profiles)} browser profile(s): {input_profiles}')
analysis_session.profile_paths = input_profiles
print((format_meta_output(
'Output name', f'{analysis_session.output_name}.{analysis_session.selected_output_format}')))
# Run the AnalysisSession
print("\n Processing:")
analysis_session.run()
print("\n Running plugins:")
log.info("Plugins:")
completed_plugins = []
# First run built-in plugins that ship with Hindsight
log.info(" Built-in Plugins:")
for plugin in pyhindsight.plugins.__all__:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
continue
log.debug(" - Loading '{}'".format(plugin))
try:
module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
except ImportError as e:
log.error(" - Error: {}".format(e))
print((format_plugin_output(plugin, "-unknown", 'import failed (see log)')))
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(analysis_session)
print((format_plugin_output(module.friendlyName, module.version, parsed_items)))
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception as e:
print((format_plugin_output(module.friendlyName, module.version, 'failed')))
log.info(" - Failed; {}".format(e))
# Then look for any custom user-provided plugins in a 'plugins' directory
log.info(" Custom Plugins:")
if real_path not in sys.path:
sys.path.insert(0, real_path)
# Loop through all paths, to pick up all potential locations for custom plugins
for potential_path in sys.path:
# If a subdirectory exists called 'plugins' or 'pyhindsight/plugins' at the current path, continue on
for potential_plugin_path in [os.path.join(potential_path, 'plugins'),
os.path.join(potential_path, 'pyhindsight', 'plugins')]:
if os.path.isdir(potential_plugin_path):
log.info(" Found custom plugin directory {}:".format(potential_plugin_path))
try:
# Insert the current plugin location to the system path, so we can import plugin modules by name
sys.path.insert(0, potential_plugin_path)
# Get list of available plugins and run them
plugin_listing = os.listdir(potential_plugin_path)
log.debug(" - Contents of plugin folder: " + str(plugin_listing))
for plugin in plugin_listing:
if plugin[-3:] == ".py" and plugin[0] != '_':
plugin = plugin.replace(".py", "")
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.debug(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
continue
log.debug(" - Loading '{}'".format(plugin))
try:
module = __import__(plugin)
except ImportError as e:
log.error(" - Error: {}".format(e))
print((format_plugin_output(plugin, "-unknown", 'import failed (see log)')))
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(analysis_session)
print((format_plugin_output(module.friendlyName, module.version, parsed_items)))
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception as e:
print((format_plugin_output(module.friendlyName, module.version, 'failed')))
log.info(" - Failed; {}".format(e))
except Exception as e:
log.debug(' - Error loading plugins ({})'.format(e))
print(' - Error loading plugins')
finally:
# Remove the current plugin location from the system path, so we don't loop over it again
sys.path.remove(potential_plugin_path)
# Check if output directory exists; attempt to create if it doesn't
if os.path.dirname(analysis_session.output_name) != "" \
and not os.path.exists(os.path.dirname(analysis_session.output_name)):
os.makedirs(os.path.dirname(analysis_session.output_name))
# Get desired output type form args.format and call the correct output creation function
if analysis_session.selected_output_format == 'xlsx':
log.info("Writing output; XLSX format selected")
try:
print(("\n Writing {}.xlsx".format(analysis_session.output_name)))
write_excel(analysis_session)
except IOError:
type, value, traceback = sys.exc_info()
print((value, "- is the file open? If so, please close it and try again."))
log.error("Error writing XLSX file; type: {}, value: {}, traceback: {}".format(type, value, traceback))
elif args.format == 'jsonl':
log.info("Writing output; JSONL format selected")
print(("\n Writing {}.jsonl".format(analysis_session.output_name)))
write_jsonl(analysis_session)
elif args.format == 'sqlite':
log.info("Writing output; SQLite format selected")
print(("\n Writing {}.sqlite".format(analysis_session.output_name)))
write_sqlite(analysis_session)
# Display and log finish time
print(f'\n Finish time: {str(datetime.datetime.now())[:-3]}')
log.info(f'Finish time: {str(datetime.datetime.now())[:-3]}\n\n')
class AnalysisSession(object):
def __init__(self,
profile_path=None,
cache_path=None,
browser_type=None,
available_input_types=None,
version=None,
display_version=None,
output_name=None,
log_path=None,
timezone=None,
available_output_formats=None,
selected_output_format=None,
available_decrypts=None,
selected_decrypts=None,
parsed_artifacts=None,
artifacts_display=None,
artifacts_counts=None,
plugin_descriptions=None,
selected_plugins=None,
plugin_results=None,
hindsight_version=None):
self.profile_path = profile_path
self.cache_path = cache_path
self.browser_type = browser_type
self.available_input_types = available_input_types
self.version = version
self.display_version = display_version
self.output_name = output_name
self.log_path = log_path
self.timezone = timezone
self.available_output_formats = available_output_formats
self.selected_output_format = selected_output_format
self.available_decrypts = available_decrypts
self.selected_decrypts = selected_decrypts
self.parsed_artifacts = parsed_artifacts
self.artifacts_display = artifacts_display
self.artifacts_counts = artifacts_counts
self.plugin_descriptions = plugin_descriptions
self.selected_plugins = selected_plugins
self.plugin_results = plugin_results
self.hindsight_version = hindsight_version
if self.available_input_types is None:
self.available_input_types = ['Chrome', 'Brave']
if self.parsed_artifacts is None:
self.parsed_artifacts = []
if self.artifacts_counts is None:
self.artifacts_counts = {}
if self.available_output_formats is None:
self.available_output_formats = ['sqlite']
if self.available_decrypts is None:
self.available_decrypts = {'windows': 0, 'mac': 0, 'linux': 0}
if self.plugin_results is None:
self.plugin_results = {}
if __version__:
self.hindsight_version = __version__
# Try to import modules for different output formats, adding to self.available_output_format array if successful
try:
import xlsxwriter
self.available_output_formats.append('xlsx')
except ImportError:
logging.warning(
"Couldn't import module 'xlsxwriter'; XLSX output disabled.")
# Set output name to default if not set by user
if self.output_name is None:
self.output_name = "Hindsight Report ({})".format(
time.strftime('%Y-%m-%dT%H-%M-%S'))
# Try to import modules for cookie decryption on different OSes.
# Windows
try:
import win32crypt
self.available_decrypts['windows'] = 1
except ImportError:
self.available_decrypts['windows'] = 0
logging.warning(
"Couldn't import module 'win32crypt'; cookie decryption on Windows disabled."
)
# Mac OS
try:
import keyring
self.available_decrypts['mac'] = 1
except ImportError:
self.available_decrypts['mac'] = 0
logging.warning(
"Couldn't import module 'keyring'; cookie decryption on Mac OS disabled."
)
# Linux / Mac OS
try:
import Cryptodome.Cipher.AES
import Cryptodome.Protocol.KDF
self.available_decrypts['linux'] = 1
except ImportError:
self.available_decrypts['linux'] = 0
self.available_decrypts['mac'] = 0
logging.warning(
"Couldn't import module 'Cryptodome'; cookie decryption on Linux/Mac OS disabled."
)
def run(self):
if self.selected_output_format is None:
self.selected_output_format = self.available_output_formats[-1]
if 'pytz' in sys.modules:
# If the timezone exists, and is a string, we need to convert it to a tzinfo object
if self.timezone is not None and isinstance(self.timezone, str):
try:
self.timezone = pytz.timezone(self.timezone)
except pytz.exceptions.UnknownTimeZoneError:
print("Couldn't understand timezone; using UTC.")
self.timezone = pytz.timezone('UTC')
elif self.timezone is None:
self.timezone = pytz.timezone('UTC')
else:
self.timezone = None
logging.debug("Options: " + str(self.__dict__))
# Analysis start time
logging.info("Starting analysis")
if self.browser_type == "Chrome":
browser_analysis = Chrome(
self.profile_path,
available_decrypts=self.available_decrypts,
cache_path=self.cache_path,
timezone=self.timezone)
browser_analysis.process()
self.parsed_artifacts = browser_analysis.parsed_artifacts
self.artifacts_counts = browser_analysis.artifacts_counts
self.artifacts_display = browser_analysis.artifacts_display
self.version = browser_analysis.version
self.display_version = browser_analysis.display_version
for item in browser_analysis.__dict__:
try:
# If the browser_analysis attribute has 'presentation' and 'data' subkeys, promote from
# browser_analysis object to analysis_session object
if browser_analysis.__dict__[item][
'presentation'] and browser_analysis.__dict__[
item]['data']:
setattr(self, item, browser_analysis.__dict__[item])
except:
pass
elif self.browser_type == "Brave":
browser_analysis = Brave(self.profile_path, timezone=self.timezone)
browser_analysis.process()
self.parsed_artifacts = browser_analysis.parsed_artifacts
self.artifacts_counts = browser_analysis.artifacts_counts
self.artifacts_display = browser_analysis.artifacts_display
self.version = browser_analysis.version
self.display_version = browser_analysis.display_version
for item in browser_analysis.__dict__:
try:
# If the browser_analysis attribute has 'presentation' and 'data' subkeys, promote from
# browser_analysis object to analysis_session object
if browser_analysis.__dict__[item][
'presentation'] and browser_analysis.__dict__[
item]['data']:
setattr(self, item, browser_analysis.__dict__[item])
except:
pass
def run_plugins(self):
logging.info("Selected plugins: " + str(self.selected_plugins))
completed_plugins = []
for plugin in self.selected_plugins:
# First check built-in plugins that ship with Hindsight
# logging.info(" Built-in Plugins:")
for standard_plugin in pyhindsight.plugins.__all__:
# Check if the standard plugin is the selected_plugin we're looking for
if standard_plugin == plugin:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
logging.info(
" - Skipping '{}'; a plugin with that name has run already"
.format(plugin))
continue
logging.info(
" - Loading '{}' [standard plugin]".format(plugin))
try:
module = importlib.import_module(
"pyhindsight.plugins.{}".format(plugin))
except ImportError, e:
logging.error(" - Error: {}".format(e))
print format_plugin_output(plugin, "-unknown",
'import failed (see log)')
continue
try:
logging.info(" - Running '{}' plugin".format(
module.friendlyName))
parsed_items = module.plugin(self)
print format_plugin_output(module.friendlyName,
module.version,
parsed_items)
self.plugin_results[plugin] = [
module.friendlyName, module.version, parsed_items
]
logging.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
break
except Exception, e:
print format_plugin_output(module.friendlyName,
module.version, 'failed')
self.plugin_results[plugin] = [
module.friendlyName, module.version, 'failed'
]
logging.info(" - Failed; {}".format(e))
for potential_path in sys.path:
# If a subdirectory exists called 'plugins' at the current path, continue on
potential_plugin_path = os.path.join(potential_path, 'plugins')
if os.path.isdir(potential_plugin_path):
try:
# Insert the current plugin location to the system path, so we can import plugin modules by name
sys.path.insert(0, potential_plugin_path)
# Get list of available plugins and run them
plugin_listing = os.listdir(potential_plugin_path)
for custom_plugin in plugin_listing:
if custom_plugin[
-3:] == ".py" and custom_plugin[0] != '_':
custom_plugin = custom_plugin.replace(
".py", "")
if custom_plugin == plugin:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
logging.info(
" - Skipping '{}'; a plugin with that name has run already"
.format(plugin))
continue
logging.debug(
" - Loading '{}' [custom plugin]".
format(plugin))
try:
module = __import__(plugin)
except ImportError, e:
logging.error(" - Error: {}".format(e))
print format_plugin_output(
plugin, "-unknown",
'import failed (see log)')
continue
try:
logging.info(
" - Running '{}' plugin".format(
module.friendlyName))
parsed_items = module.plugin(self)
print format_plugin_output(
module.friendlyName,
module.version, parsed_items)
self.plugin_results[plugin] = [
module.friendlyName,
module.version, parsed_items
]
logging.info(" - Completed; {}".format(
parsed_items))
completed_plugins.append(plugin)
except Exception, e:
print format_plugin_output(
module.friendlyName,
module.version, 'failed')
self.plugin_results[plugin] = [
module.friendlyName,
module.version, 'failed'
]
logging.info(" - Failed; {}".format(e))
except Exception as e:
logging.debug(
' - Error loading plugins ({})'.format(e))
print ' - Error loading plugins'
def main():
def write_excel(analysis_session):
import StringIO
# Set up a StringIO object to save the XLSX content to before saving to disk
string_buffer = StringIO.StringIO()
# Generate the XLSX content using the function in the AnalysisSession and save it to the StringIO object
analysis_session.generate_excel(string_buffer)
# Go back to the beginning (be kind, rewind)
string_buffer.seek(0)
# Write the StringIO object to a file on disk named what the user specified
with open("{}.{}".format(os.path.join(real_path, analysis_session.output_name), analysis_session.selected_output_format), 'wb') as file_output:
shutil.copyfileobj(string_buffer, file_output)
def write_sqlite(analysis_session):
output_file = analysis_session.output_name + '.sqlite'
if os.path.exists(output_file):
if os.path.getsize(output_file) > 0:
print('\nDatabase file "{}" already exists.\n'.format(output_file))
user_input = raw_input('Would you like to (O)verwrite it, (R)ename output file, or (E)xit? ')
over_re = re.compile(r'(^o$|overwrite)', re.IGNORECASE)
rename_re = re.compile(r'(^r$|rename)', re.IGNORECASE)
exit_re = re.compile(r'(^e$|exit)', re.IGNORECASE)
if re.search(exit_re, user_input):
print("Exiting... ")
sys.exit()
elif re.search(over_re, user_input):
os.remove(output_file)
print("Deleted old \"%s\"" % output_file)
elif re.search(rename_re, user_input):
output_file = "{}_1.sqlite".format(output_file[:-7])
print("Renaming new output to {}".format(output_file))
else:
print("Did not understand response. Exiting... ")
sys.exit()
analysis_session.generate_sqlite(output_file)
def write_jsonl(analysis_session):
output_file = analysis_session.output_name + '.jsonl'
analysis_session.generate_jsonl(output_file)
print(banner)
# Useful when Hindsight is run from a different directory than where the file is located
real_path = os.path.dirname(os.path.realpath(sys.argv[0]))
# Set up the AnalysisSession object, and transfer the relevant input arguments to it
analysis_session = AnalysisSession()
# parse_arguments needs the analysis_session as an input to set things like available decrypts
args = parse_arguments(analysis_session)
if args.output:
analysis_session.output_name = args.output
if args.cache:
analysis_session.cache_path = args.cache
analysis_session.selected_output_format = args.format
analysis_session.browser_type = args.browser_type
analysis_session.timezone = args.timezone
if args.log == 'hindsight.log':
args.log = os.path.join(real_path, args.log)
analysis_session.log_path = args.log
# Set up logging
logging.basicConfig(filename=analysis_session.log_path, level=logging.DEBUG,
format='%(asctime)s.%(msecs).03d | %(levelname).01s | %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
log = logging.getLogger(__name__)
# Hindsight version info
log.info(
'\n' + '#' * 80 + '\n### Hindsight v{} (https://github.com/obsidianforensics/hindsight) ###\n'
.format(pyhindsight.__version__) + '#' * 80)
# Analysis start time
print(format_meta_output("Start time", str(datetime.datetime.now())[:-3]))
# Read the input directory
analysis_session.input_path = args.input
print(format_meta_output("Input directory", args.input))
log.info("Reading files from %s" % args.input)
input_listing = os.listdir(args.input)
log.debug("Input directory contents: " + str(input_listing))
# Search input directory for browser profiles to analyze
input_profiles = analysis_session.find_browser_profiles(args.input)
log.info(" - Found {} browser profile(s): {}".format(len(input_profiles), input_profiles))
analysis_session.profile_paths = input_profiles
print(format_meta_output("Output name", "{}.{}".format(analysis_session.output_name, analysis_session.selected_output_format)))
# Run the AnalysisSession
print("\n Processing:")
analysis_session.run()
print("\n Running plugins:")
log.info("Plugins:")
completed_plugins = []
# First run built-in plugins that ship with Hindsight
log.info(" Built-in Plugins:")
for plugin in pyhindsight.plugins.__all__:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
continue
log.debug(" - Loading '{}'".format(plugin))
try:
module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
except ImportError, e:
log.error(" - Error: {}".format(e))
print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(analysis_session)
print(format_plugin_output(module.friendlyName, module.version, parsed_items))
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception, e:
print(format_plugin_output(module.friendlyName, module.version, 'failed'))
log.info(" - Failed; {}".format(e))
log.debug(" - Contents of plugin folder: " + str(plugin_listing))
for plugin in plugin_listing:
if plugin[-3:] == ".py" and plugin[0] != '_':
plugin = plugin.replace(".py", "")
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.debug(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
continue
log.debug(" - Loading '{}'".format(plugin))
try:
module = __import__(plugin)
except ImportError, e:
log.error(" - Error: {}".format(e))
print(format_plugin_output(plugin, "-unknown", 'import failed (see log)'))
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(analysis_session)
print(format_plugin_output(module.friendlyName, module.version, parsed_items))
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception, e:
print(format_plugin_output(module.friendlyName, module.version, 'failed'))
log.info(" - Failed; {}".format(e))
except Exception as e:
log.debug(' - Error loading plugins ({})'.format(e))
print(' - Error loading plugins')
finally:
# Remove the current plugin location from the system path, so we don't loop over it again
class AnalysisSession(object):
def __init__(self, input_path=None, profile_paths=None, cache_path=None, browser_type=None, available_input_types=None,
version=None, display_version=None, output_name=None, log_path=None, timezone=None,
available_output_formats=None, selected_output_format=None, available_decrypts=None,
selected_decrypts=None, parsed_artifacts=None, artifacts_display=None, artifacts_counts=None,
plugin_descriptions=None, selected_plugins=None, plugin_results=None, hindsight_version=None, preferences=None):
self.input_path = input_path
self.profile_paths = profile_paths
self.cache_path = cache_path
self.browser_type = browser_type
self.available_input_types = available_input_types
self.version = version
self.display_version = display_version
self.output_name = output_name
self.log_path = log_path
self.timezone = timezone
self.available_output_formats = available_output_formats
self.selected_output_format = selected_output_format
self.available_decrypts = available_decrypts
self.selected_decrypts = selected_decrypts
self.parsed_artifacts = parsed_artifacts
self.artifacts_display = artifacts_display
self.artifacts_counts = artifacts_counts
self.plugin_descriptions = plugin_descriptions
self.selected_plugins = selected_plugins
self.plugin_results = plugin_results
self.hindsight_version = hindsight_version
self.preferences = preferences
if self.version is None:
self.version = []
if self.available_input_types is None:
self.available_input_types = ['Chrome', 'Brave']
if self.parsed_artifacts is None:
self.parsed_artifacts = []
if self.artifacts_counts is None:
self.artifacts_counts = {}
if self.available_output_formats is None:
self.available_output_formats = ['sqlite']
if self.available_decrypts is None:
self.available_decrypts = {'windows': 0, 'mac': 0, 'linux': 0}
if self.plugin_results is None:
self.plugin_results = {}
if self.preferences is None:
self.preferences = []
if __version__:
self.hindsight_version = __version__
# Try to import modules for different output formats, adding to self.available_output_format array if successful
try:
import xlsxwriter
self.available_output_formats.append('xlsx')
except ImportError:
log.warning("Couldn't import module 'xlsxwriter'; XLSX output disabled.")
# Set output name to default if not set by user
if self.output_name is None:
self.output_name = "Hindsight Report ({})".format(time.strftime('%Y-%m-%dT%H-%M-%S'))
# Try to import modules for cookie decryption on different OSes.
# Windows
try:
import win32crypt
self.available_decrypts['windows'] = 1
except ImportError:
self.available_decrypts['windows'] = 0
# Mac OS
try:
import keyring
self.available_decrypts['mac'] = 1
except ImportError:
self.available_decrypts['mac'] = 0
# Linux / Mac OS
try:
import Cryptodome.Cipher.AES
import Cryptodome.Protocol.KDF
self.available_decrypts['linux'] = 1
except ImportError:
self.available_decrypts['linux'] = 0
self.available_decrypts['mac'] = 0
@staticmethod
def sum_dict_counts(dict1, dict2):
"""Combine two dicts by summing the values of shared keys"""
for key, value in dict2.items():
# Case 1: dict2's value for key is a string (aka: it failed)
if isinstance(value, str):
# The value should only be non-int if it's a Failed message
if not value.startswith('Fail'):
raise ValueError('Unexpected status value')
dict1[key] = dict1.setdefault(key, 0)
# Case 2: dict1's value of key is a string (aka: it failed)
elif isinstance(dict1.get(key), str):
# The value should only be non-int if it's a Failed message
if not dict1.get(key).startswith('Fail'):
raise ValueError('Unexpected status value')
dict1[key] = value
# Case 3: dict2's value for key is an int, or doesn't exist.
else:
dict1[key] = dict1.setdefault(key, 0) + value
return dict1
def promote_object_to_analysis_session(self, item_name, item_value):
if self.__dict__.get(item_name):
self.__dict__[item_name]['data'].extend(item_value['data'])
# TODO: add some checks around value of presentation. It *shouldn't* differ...
self.__dict__[item_name]['presentation'] = item_value['presentation']
else:
setattr(self, item_name, item_value)
@staticmethod
def is_profile(base_path, existing_files, warn=False):
"""
Log a warning message if any file in `required_files` is missing from
`existing_files`. Return True if all required files are present.
"""
is_profile = True
for required_file in ['History', 'Cookies']:
# This approach (checking the file names) is naive but should work.
if required_file not in existing_files:
if warn:
log.warning("The profile directory {} does not contain the file {}. Analysis may not be very useful.")
is_profile = False
return is_profile
def search_subdirs(self, base_path):
"""Recursively search a path for browser profiles"""
found_profile_paths = []
base_dir_listing = os.listdir(base_path)
if self.is_profile(base_path, base_dir_listing):
found_profile_paths.append(base_path)
for item in base_dir_listing:
item_path = os.path.join(base_path, item)
if os.path.isdir(item_path):
profile_found_in_subdir = self.search_subdirs(item_path)
if profile_found_in_subdir:
found_profile_paths.extend(profile_found_in_subdir)
return found_profile_paths
def find_browser_profiles(self, base_path):
"""Search a path for browser profiles (only Chromium-based at the moment)."""
found_profile_paths = [base_path]
base_dir_listing = os.listdir(base_path)
# The 'History' and 'Cookies' SQLite files are kind of the minimum required for most
# Chrome analysis. Warn if they are not present.
if not self.is_profile(base_path, base_dir_listing, warn=True):
# Only search sub dirs if the current dir is not a Profile (Profiles are not nested).
found_profile_paths.extend(self.search_subdirs(base_path))
log.debug("Profile paths: " + str(found_profile_paths))
return found_profile_paths
def generate_display_version(self):
self.version = sorted(self.version)
if self.version[0] != self.version[-1]:
self.display_version = "%s-%s" % (self.version[0], self.version[-1])
else:
self.display_version = self.version[0]
def run(self):
if self.selected_output_format is None:
self.selected_output_format = self.available_output_formats[-1]
if 'pytz' in sys.modules:
# If the timezone exists, and is a string, we need to convert it to a tzinfo object
if self.timezone is not None and isinstance(self.timezone, str):
try:
self.timezone = pytz.timezone(self.timezone)
except pytz.exceptions.UnknownTimeZoneError:
print("Couldn't understand timezone; using UTC.")
self.timezone = pytz.timezone('UTC')
elif self.timezone is None:
self.timezone = pytz.timezone('UTC')
else:
self.timezone = None
log.debug("Options: " + str(self.__dict__))
# Analysis start time
log.info("Starting analysis")
# Search input directory for browser profiles to analyze
input_profiles = self.find_browser_profiles(self.input_path)
log.info(" - Found {} browser profile(s): {}".format(len(input_profiles), input_profiles))
self.profile_paths = input_profiles
# Make sure the input is what we're expecting
assert isinstance(self.profile_paths, list)
assert len(self.profile_paths) >= 1
for found_profile_path in self.profile_paths:
if self.browser_type == "Chrome":
browser_analysis = Chrome(found_profile_path, available_decrypts=self.available_decrypts,
cache_path=self.cache_path, timezone=self.timezone)
browser_analysis.process()
self.parsed_artifacts.extend(browser_analysis.parsed_artifacts)
self.artifacts_counts = self.sum_dict_counts(self.artifacts_counts, browser_analysis.artifacts_counts)
self.artifacts_display = browser_analysis.artifacts_display
self.version.extend(browser_analysis.version)
self.display_version = browser_analysis.display_version
self.preferences.extend(browser_analysis.preferences)
for item in browser_analysis.__dict__:
if isinstance(browser_analysis.__dict__[item], dict):
try:
# If the browser_analysis attribute has 'presentation' and 'data' subkeys, promote from
if browser_analysis.__dict__[item].get('presentation') and browser_analysis.__dict__[item].get('data'):
self.promote_object_to_analysis_session(item, browser_analysis.__dict__[item])
except Exception as e:
log.info("Exception occurred while analyzing {} for analysis session promotion: {}".format(item, e))
elif self.browser_type == "Brave":
browser_analysis = Brave(found_profile_path, timezone=self.timezone)
browser_analysis.process()
self.parsed_artifacts = browser_analysis.parsed_artifacts
self.artifacts_counts = browser_analysis.artifacts_counts
self.artifacts_display = browser_analysis.artifacts_display
self.version = browser_analysis.version
self.display_version = browser_analysis.display_version
for item in browser_analysis.__dict__:
if isinstance(browser_analysis.__dict__[item], dict):
try:
# If the browser_analysis attribute has 'presentation' and 'data' subkeys, promote from
if browser_analysis.__dict__[item].get('presentation') and browser_analysis.__dict__[item].get('data'):
self.promote_object_to_analysis_session(item, browser_analysis.__dict__[item])
except Exception as e:
log.info("Exception occurred while analyzing {} for analysis session promotion: {}".format(item, e))
self.generate_display_version()
def run_plugins(self):
log.info("Selected plugins: " + str(self.selected_plugins))
completed_plugins = []
for plugin in self.selected_plugins:
# First check built-in plugins that ship with Hindsight
# log.info(" Built-in Plugins:")
for standard_plugin in pyhindsight.plugins.__all__:
# Check if the standard plugin is the selected_plugin we're looking for
if standard_plugin == plugin:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.info(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
continue
log.info(" - Loading '{}' [standard plugin]".format(plugin))
try:
module = importlib.import_module("pyhindsight.plugins.{}".format(plugin))
except ImportError, e:
log.error(" - Error: {}".format(e))
print format_plugin_output(plugin, "-unknown", 'import failed (see log)')
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(self)
print format_plugin_output(module.friendlyName, module.version, parsed_items)
self.plugin_results[plugin] = [module.friendlyName, module.version, parsed_items]
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
break
except Exception, e:
print format_plugin_output(module.friendlyName, module.version, 'failed')
self.plugin_results[plugin] = [module.friendlyName, module.version, 'failed']
log.info(" - Failed; {}".format(e))
for potential_path in sys.path:
# If a subdirectory exists called 'plugins' at the current path, continue on
potential_plugin_path = os.path.join(potential_path, 'plugins')
if os.path.isdir(potential_plugin_path):
try:
# Insert the current plugin location to the system path, so we can import plugin modules by name
sys.path.insert(0, potential_plugin_path)
# Get list of available plugins and run them
plugin_listing = os.listdir(potential_plugin_path)
for custom_plugin in plugin_listing:
if custom_plugin[-3:] == ".py" and custom_plugin[0] != '_':
custom_plugin = custom_plugin.replace(".py", "")
if custom_plugin == plugin:
# Check to see if we've already run this plugin (likely from a different path)
if plugin in completed_plugins:
log.info(" - Skipping '{}'; a plugin with that name has run already".format(plugin))
continue
log.debug(" - Loading '{}' [custom plugin]".format(plugin))
try:
module = __import__(plugin)
except ImportError, e:
log.error(" - Error: {}".format(e))
print format_plugin_output(plugin, "-unknown", 'import failed (see log)')
continue
try:
log.info(" - Running '{}' plugin".format(module.friendlyName))
parsed_items = module.plugin(self)
print format_plugin_output(module.friendlyName, module.version, parsed_items)
self.plugin_results[plugin] = [module.friendlyName, module.version, parsed_items]
log.info(" - Completed; {}".format(parsed_items))
completed_plugins.append(plugin)
except Exception, e:
print format_plugin_output(module.friendlyName, module.version, 'failed')
self.plugin_results[plugin] = [module.friendlyName, module.version, 'failed']
log.info(" - Failed; {}".format(e))
except Exception as e:
log.debug(' - Error loading plugins ({})'.format(e))
print ' - Error loading plugins'