def run(self, path=""): self.search = GetFiles(self.config, vss=self.myflag("vss")) self.vss = self.myflag('vss') self.logger().info("Parsing ShimCache from registry") outfolder = self.myconfig('voutdir') if self.vss else self.myconfig( 'outdir') SYSTEM = list(self.search.search(r"windows/System32/config/SYSTEM$")) check_directory(outfolder, create=True) partition_list = set() for f in SYSTEM: aux = re.search(r"([vp\d]*)/windows/System32/config", f, re.I) partition_list.add(aux.group(1)) output_files = { p: os.path.join(outfolder, "shimcache_%s.csv" % p) for p in partition_list } for f in SYSTEM: save_csv(self.parse_ShimCache_hive(f), outfile=output_files[f.split("/")[2]], file_exists='OVERWRITE', quoting=0) self.logger().info("Finished extraction from ShimCache") return []
def run(self, path=""): vss = self.myflag('vss') self.search = GetFiles(self.config, vss=vss) outfolder = self.myconfig('voutdir') if vss else self.myconfig( 'outdir') check_directory(outfolder, create=True) amcache_hives = [path] if path else self.search.search("Amcache.hve$") for am_file in amcache_hives: self.amcache_path = os.path.join(self.myconfig('casedir'), am_file) partition = am_file.split("/")[2] self.logger().info("Parsing {}".format(am_file)) self.outfile = os.path.join(outfolder, "amcache_{}.csv".format(partition)) try: reg = Registry.Registry( os.path.join(self.myconfig('casedir'), am_file)) entries = self.parse_amcache_entries(reg) save_csv(entries, outfile=self.outfile, file_exists='OVERWRITE', quoting=0) except KeyError: self.logger().warning( "Expected subkeys not found in hive file: {}".format( am_file)) except Exception as exc: self.logger().warning("Problems parsing: {}. Error: {}".format( am_file, exc)) self.logger().info("Amcache.hve parsing finished") return []
def run(self, path): self.check_params(path, check_path=True) output_dir = self.myconfig('outdir') check_directory(output_dir, create=True) if not os.path.isabs(path): path = os.path.join(self.myconfig('casedir'), path) try: frame_rate = self._snapshowFrequency(path) export_path = os.path.join( output_dir, relative_path(path, self.myconfig('casedir'))) self.logger().debug('path=%s frame_rate=%s export_path=%s', path, frame_rate, export_path) check_directory(export_path, create=True) base.commands.run_command( r'ffmpeg -loglevel quiet -i "{}" -vf fps={} "{}"'.format( path, frame_rate, os.path.join(export_path, 'img%03d.jpg'))) # subprocess.call(['ffmpeg', '-i', filename, '-vf', 'fps=%s' % frame_rate, r'img%03d.jpg']) return [ dict(path=relative_path(path, self.myconfig('casedir')), preview=list( map( lambda p: relative_path( os.path.join(export_path, p), self.myconfig('casedir')), os.listdir(export_path)))) ] except Exception as exc: self.logger().warning( 'Cannot create snapshots from path=%s exc=%s', path, exc) return []
def run(self, path=""): """ Extracts SRUM artifacts of a disk """ vss = self.myflag('vss') SRUM_TEMPLATE = os.path.join(self.myconfig('rvthome'), "plugins/external/srum-dump/SRUM_TEMPLATE2.xlsx") srum = os.path.join(self.myconfig('rvthome'), "plugins/external/srum-dump/srum_dump2.py") check_file(SRUM_TEMPLATE, error_missing=True) Search = GetFiles(self.config, vss=self.myflag("vss")) SOFTWARE = list(Search.search('windows/system32/config/SOFTWARE$')) SRUDB = list(Search.search('/windows/system32/sru/SRUDB.dat$')) python3 = os.path.join(self.myconfig('rvthome'), ".venv/bin/python3") out_folder = self.myconfig('voutdir') if vss else self.myconfig('outdir') check_directory(out_folder, create=True) if not SRUDB: self.logger().info("SRUDB.dat not found in any partition of the disk") return [] for soft in SOFTWARE: partition = soft.split('/')[2] for srudb in SRUDB: if srudb.split('/')[2] == partition: self.logger().info("Parsing SRUDB from partition {}".format(partition)) out_file = os.path.join(out_folder, 'srum_{}.xlsx'.format(partition)) run_command([python3, srum, "-i", os.path.join(self.myconfig('casedir'), srudb), "-t", SRUM_TEMPLATE, "-r", os.path.join(self.myconfig('casedir'), soft), "-o", out_file], logger=self.logger()) self.convert_to_csv(out_folder, partition) os.remove(out_file) break else: self.logger().info("SRUDB.dat not found in partition: {}".format(partition)) return []
def __init__(self, *args, disk=None, **kwargs): super().__init__(*args, **kwargs) self.disk = disk if disk is None: self.disk = getSourceImage(self.myconfig) self.string_path = self.myconfig('outdir') check_directory(self.string_path, create=True)
def parse_BITS(self): if self.vss: base_path = self.myconfig('voutdir') bitsdb = self.search.search( r"v\d+p\d+/ProgramData/Microsoft/Network/Downloader/qmgr0.dat$" ) else: base_path = self.myconfig('outdir') bitsdb = self.search.search( r"p\d+/ProgramData/Microsoft/Network/Downloader/qmgr0.dat$") check_directory(base_path, create=True) fields = OrderedDict([('job_id', None), ('name', None), ('desc', None), ('type', None), ('priority', None), ('sid', None), ('state', None), ('cmd', None), ('args', None), ('file_count', 0), ('file_id', 0), ('dest_fn', None), ('src_fn', None), ('tmp_fn', None), ('download_size', -1), ('transfer_size', -1), ('drive', None), ('vol_guid', None), ('ctime', None), ('mtime', None), ('other_time0', None), ('other_time1', None), ('other_time2', None), ('carved', False)]) for f in bitsdb: analyzer = bits.Bits.load_file( os.path.join(self.myconfig('casedir'), f)) jobs = analyzer.parse() res_generator = (OrderedDict([(field, j.get(field, fields[field])) for field in fields]) for j in jobs) output_file = os.path.join(base_path, "bitsdb_%s.csv" % f.split("/")[2]) save_csv(res_generator, outfile=output_file, file_exists='OVERWRITE')
def parse_SysCache_hive(self): outfolder = self.myconfig('voutdir') if self.vss else self.myconfig( 'outdir') # self.tl_file = os.path.join(self.myconfig('timelinesdir'), "%s_BODY.csv" % self.myconfig('source')) check_directory(outfolder, create=True) SYSC = self.search.search(r"/System Volume Information/SysCache.hve$") ripcmd = self.config.get('plugins.common', 'rip', '/opt/regripper/rip.pl') for f in SYSC: p = f.split('/')[2] output_text = run_command([ ripcmd, "-r", os.path.join(self.myconfig('casedir'), f), "-p", "syscache_csv" ], logger=self.logger()) output_file = os.path.join(outfolder, "syscache_%s.csv" % p) self.path_from_inode = FileSystem( config=self.config).load_path_from_inode(self.myconfig, p, vss=self.vss) save_csv(self.parse_syscache_csv(p, output_text), outfile=output_file, file_exists='OVERWRITE') self.logger().info("Finished extraction from SysCache")
def run(self, path=""): """ Main function to generate report files """ vss = self.myflag('vss') check_directory(self.myconfig('mountdir'), error_missing=True) for p in os.listdir(self.myconfig('mountdir')): # parse only partition directories if (p.startswith('p') and not vss) or (p.startswith('v') and vss): regfiles = self.get_hives(p) self.generate_registry_output(p, regfiles) return []
def run(self, path=""): """ Main function to extract $Recycle.bin files. """ if self.vss: output_path = self.myconfig('voutdir') else: output_path = self.myconfig('outdir') try: check_file(self.timeline_file, error_missing=True) except base.job.RVTError: return [] check_directory(output_path, create=True) self.filesystem = FileSystem(self.config) # Get the users associated with each SID for every partition self.sid_user = {} if self.vss: for p in self.vss_partitions: self.sid_user[p] = self.generate_SID_user(p) else: for p in self.partitions: self.sid_user[p] = self.generate_SID_user(p) self.logger().info('Starting to parse RecycleBin') # RB_codes relates a a six digit recyclebin code with a path for a file. Are updated for each partition or vss? self.RB_codes = {} if self.vss: for partition in self.vss_partitions: self.logger().info( 'Processing Recycle Bin in partition {}'.format(partition)) try: self.parse_RecycleBin(partition) except Exception as exc: if self.myflag('stop_on_error'): raise exc continue output_file = os.path.join( output_path, "{}_recycle_bin.csv".format(partition)) self.save_recycle_files(output_file, partition, sorting=True) else: try: self.parse_RecycleBin() except Exception as exc: if self.myflag('stop_on_error'): raise exc return [] output_file = os.path.join(output_path, "recycle_bin.csv") self.save_recycle_files(output_file, sorting=True) self.logger().info("Done parsing Recycle Bin!") return []
def run(self, path=""): vss = self.myflag('vss') self.indir = self.config.get('plugins.windows.RVT_evtx.Evtx', 'voutdir') if vss else self.config.get('plugins.windows.RVT_evtx.Evtx', 'outdir') self.outdir = self.myconfig('voutdir') if vss else self.myconfig('outdir') check_directory(self.indir, error_missing=True) check_directory(self.outdir, create=True) if not vss: self.main() else: for dir in os.listdir(self.indir): self.main(dir) return []
def run(self, path=""): self.vss = self.myflag('vss') self.search = GetFiles(self.config, vss=self.myflag("vss")) self.outfolder = self.myconfig( 'voutdir') if self.vss else self.myconfig('outdir') check_directory(self.outfolder, create=True) self.logger().info( "Parsing artifacts from scheduled tasks files (.job)") self.parse_Task() self.logger().info( "Parsing artifacts from Task Scheduler Service log files (schedlgu.txt)" ) self.parse_schedlgu() return []
def run(self, keyfile=""): """ Searche contents of regex in output dir except in strings, searches and parser folders """ self.logger().info("Searching at output folder") if not keyfile: keyfile = self.myconfig('keyfile') check_file(keyfile, error_missing=True) grep = self.config.get('plugins.common', 'grep', '/bin/grep') skip_folders = ("strings", "parser", "searches") self.logger().info("Getting key list from {}".format(keyfile)) keywords = getSearchItems(keyfile) temp_dir = tempfile.mkdtemp('outsearch') outdir = self.myconfig('outdir') check_directory(outdir, create=True) for kw, srch in keywords.items(): output_file = os.path.join(temp_dir, "outsearch_{}.txt".format(kw)) with open(output_file, "w") as f: f.write( "\nKeyword: {}\n-----------------------------\n\n".format( srch)) f.flush() for item in os.listdir(self.myconfig('outputdir')): folder = os.path.join(self.myconfig('outputdir'), item) if os.path.isdir(folder) and item not in skip_folders: run_command([grep, "-ilR", srch, item], stdout=f, from_dir=self.myconfig('outputdir'), logger=self.logger()) try: for file in os.listdir(temp_dir): shutil.copy(os.path.join(temp_dir, file), os.path.join(outdir, file)) finally: shutil.rmtree(temp_dir) self.logger().info("OutSearch done") return []
def run(self, path=""): self.disk = getSourceImage(self.myconfig) keyfile = path self.logger().debug('Testing existance of {}'.format(keyfile)) if not keyfile: keyfile = self.myconfig('keyfile') check_file(keyfile, error_missing=True) # Get string files or generate them if not found self.string_path = self.myconfig('strings_dir') if not (check_directory(self.string_path) and os.listdir(self.string_path)): self.logger().debug("No string files found. Generating them") StringGenerate(config=self.config, disk=self.disk).generate_strings() self.search_path = self.myconfig('outdir') check_directory(self.search_path, create=True) self.keywords = getSearchItems( keyfile) # Get kw:regex dictionary reading keyfile self.blocks = { } # Store set of blocks for kw and partition. Ex: {'my_kw': {'p02': set(1234, 1235, ...)}} self.block_status = defaultdict( dict ) # Store status for blocks with search hits in a partition. Ex:{'03':{4547:'Allocated', 1354536:'Not Allocated'}} self.fs_object = FileSystem(self.config, disk=self.disk) # Generate or load 'hits_' and 'blocks_' files for kname in tqdm(self.keywords, total=len(self.keywords), desc='Searching keywords in strings'): kw = kname.strip() self.get_blocks(kw, self.keywords[kname]) # Generate 'all_' files self.get_cluster() self.logger().info("StringSearch done") return []
def generate(self, evtx_path): """ Auxiliary function """ check_directory(evtx_path, create=True) evtx = self.config.get('plugins.common', 'evtxdump', '/usr/local/bin/evtxdump.pl') alloc_files = GetFiles(self.config, vss=self.myflag("vss")) if self.vss: evtx_files = alloc_files.search(r"{}.*\.evtx$".format( evtx_path.split('/')[-1])) else: evtx_files = alloc_files.search(r"\.evtx$") errorlog = self.myconfig( 'errorlog', os.path.join(self.myconfig('sourcedir'), "{}_aux.log".format(self.myconfig('source')))) for i in evtx_files: evtx_file = os.path.join(self.myconfig('casedir'), i) if not check_file(evtx_file): self.logger().warning('File %s does not exist', evtx_file) continue self.logger().info("Parsing {}".format(i)) name = os.path.join(evtx_path, os.path.basename(i))[:-4] + "txt" # if the output already exists, continue if check_file(name): self.logger().debug( 'The output file %s ready exists. Skipping', name) continue with open(name, "wb") as f: with open(errorlog, 'a') as logfile: run_command([evtx, evtx_file], stdout=f, stderr=logfile, logger=self.logger())
def run(self, path=""): """ Creates a report based on the output of LnkExtract. """ vss = self.myflag('vss') self.logger().info("Generating lnk files report") self.mountdir = self.myconfig('mountdir') lnk_path = self.config.get('plugins.windows.RVT_lnk.LnkExtract', '{}outdir'.format('v' * vss)) report_lnk_path = self.myconfig('{}outdir'.format('v' * vss)) check_directory(lnk_path, error_missing=True) check_folder(report_lnk_path) outfile = os.path.join(report_lnk_path, 'recentfiles.csv') save_csv(self.report_recent(lnk_path), config=self.config, outfile=outfile, quoting=0) return []
def report_search_kw(self, keyword, regex): """ Creates a pdf file from 'all_kw' file, using LaTex. Parameters: keyword (str): keyword name regex (str): regular expression associated to keyword """ # TODO: do not break lines. Use lstlisting or something else pdflatex = self.myconfig('pdflatex', '/usr/bin/pdflatex') search_path = self.myconfig('search_dir') check_directory(search_path, error_missing=True) report_path = self.myconfig('outdir') check_directory(report_path, create=True) kw_utf8 = ''.join([i + '.' for i in keyword]) # Avoid LaTeX special characters replaces = [(u'\ufffd', "."), ("\\", "/"), (r"{", "("), (r"]", ")"), (r"$", "\\$"), (r"_", "\\_"), (r"%", "\\%"), (r"}", ")"), (r"^", "."), (r"#", "\\#"), (r"~", "."), ("&", "\\&"), ('"', "'"), (r"€", "euro")] line_width = 68 # number of characters per line in tex file for file in os.listdir(search_path): if not file.startswith("all_{}".format(keyword)): continue self.logger().info('Creating file {}'.format(file + '.pdf')) with open(os.path.join(report_path, file + ".tex"), "w") as foutput: foutput.write( "\\documentclass[a4paper,11pt,oneside]{report}\n\\usepackage[spanish]{babel}\n" ) foutput.write("\\usepackage[utf8]{inputenc}\n") foutput.write("\\usepackage[pdftex]{color,graphicx}\n") foutput.write("\\usepackage[pdftex,colorlinks]{hyperref}\n") foutput.write("\\usepackage{fancyvrb}\n") foutput.write("\\usepackage{eurosym}\n") foutput.write("\\usepackage{listings}\n") foutput.write( "\\lstset{breakatwhitespace=false,breaklines=true,frame=single}\n" ) foutput.write("\\UseRawInputEncoding\n") foutput.write("\\begin{document}\n\n") foutput.write( "\\section*{blindsearches in disk. Keyword: \\emph{" + keyword + "}}\n") initial = True if os.path.getsize(os.path.join(search_path, file)) == 0: foutput.write("\\end{document}\n") continue with open(os.path.join(search_path, file), "rb") as finput: for line in finput: line = line.decode("iso8859-15", "replace") for r in replaces: line = line.replace(r[0], r[1]) if line.startswith('Pt: p'): # Block information foutput.write( "\\end{Verbatim}\n\n" if not initial else "") foutput.write("\\newpage\n" if not initial else "") initial = False foutput.write("\\begin{lstlisting}\n") foutput.write(line) foutput.write("\\end{lstlisting}\n") foutput.write( "\\begin{Verbatim}[commandchars=\\\\\\{\\}]\n") continue line = re.sub("[\x00-\x09\x0B-\x1F\x7F-\xFF]", ".", line) # Write by chuncks. Note: Some hits may be missed this way for chunk_line in [ line[i:i + line_width] for i in range(0, len(line), line_width) ]: chunk_line = re.sub('({})'.format(regex), r"\\colorbox{green}{" + r'\1' + r"}", chunk_line, flags=re.I | re.M) chunk_line = re.sub('({})'.format(kw_utf8), r"\\colorbox{green}{" + r'\1' + r"}", chunk_line, flags=re.I | re.M) foutput.write(chunk_line + "\n") foutput.write("\\end{Verbatim}\n") foutput.write("\\end{document}\n") run_command( [pdflatex, "-output-directory", report_path, file + ".tex"], logger=self.logger()) break else: self.logger().warning( 'No file: all_{}. Perhaps there is no match for the keyword'. format(keyword)) for file in os.listdir(report_path): if file.endswith(".log") or file.endswith(".tex") or file.endswith( ".aux") or file.endswith(".toc") or file.endswith( ".out") or file.endswith(".synctex.gz"): os.remove(os.path.join(report_path, file))
def run(self, path=""): """ Generator of INDX entries as dictionaries. Also writes to csv files""" self.disk = getSourceImage(self.myconfig) self.sector_size = self.disk.sectorsize self.parseINDX_ROOTFiles = self.myflag( 'root', False) # Parse also INDX_ROOT records if set self.skip_short_filenames = self.myflag('skip_short', False) self.only_slack = self.myflag('only_slack', False) outdir = self.myconfig('outdir') check_directory(outdir, create=True) for p in self.disk.partitions: if not p.isMountable: continue # Get a dictionary {inode: list of names} from 'fls' to later relate inodes to a path. 'inode' keys are strings, not int. part_name = ''.join(['p', p.partition]) try: self.inode_fls = FileSystem( self.config).load_path_from_inode(partition=part_name) self.logger().debug( 'Correctly loaded inode-name relation file for partiton {}' .format(part_name)) except Exception as e: self.logger().error(e) continue # Start the carving at next to last execution block parsed outfile = os.path.join( outdir, '{}{}_INDX_timeline.csv'.format( part_name, '_slack' if self.only_slack else '')) self.lastParsedBlk = 0 if self.myflag('use_localstore'): self.lastParsedBlk = int( self.config.store_get( 'last_{}_block_parsed'.format(part_name), 0)) self.logger().debug('lastParsedBlk: {}'.format(self.lastParsedBlk)) csv_args = {'file_exists': 'APPEND', 'write_header': True} if self.lastParsedBlk: if not os.path.exists(outfile): self.logger().warning( 'Starting new file {0} at an advanced offset. Set "last_{0}_block_parsed" at 0 in "store.ini" if a fresh start is desired' .format(outfile)) else: csv_args['write_header'] = False else: if os.path.exists(outfile): self.logger().warning( 'Overwriting file {}'.format(outfile)) csv_args['file_exists'] = 'OVERWRITE' # Write the parsed entries to a csv file for each partition. save_csv(self.parse_INDX(p), config=self.config, outfile=outfile, quoting=0, **csv_args) return []
def characterize_Linux(self): """ """ self.outfile = self.myconfig('outfile') check_directory(os.path.dirname(self.outfile), create=True) for p in self.disk.partitions: part_path = os.path.join(self.myconfig('mountdir'), "p%s" % p.partition) if not os.path.isdir(os.path.join(part_path, "etc")): continue releas_f = "" if os.path.isfile(os.path.join( part_path, "etc/lsb-release")) or os.path.islink( os.path.join(part_path, "etc/lsb-release")): releas_f = os.path.join(part_path, "etc/lsb-release") if os.path.islink(releas_f): releas_f = os.path.join(part_path, os.path.realpath(releas_f)[1:]) else: for f in os.listdir(os.path.join(part_path, "etc")): if f.endswith("-release"): releas_f = os.path.join(part_path, "etc", f) with open(self.outfile, 'w') as out_f: if releas_f != "": out_f.write("Information of partition {}\n\n".format( p.partition)) f_rel = open(releas_f, "r") dist_id = f_rel.readline().split("=")[-1].rstrip() dist_rel = f_rel.readline().split("=")[-1].rstrip() dist_coden = f_rel.readline().split("=")[-1].rstrip() dist_desc = f_rel.readline().split("=")[-1].rstrip() kernel_v = "" f_hostname = open(os.path.join(part_path, "etc/hostname"), "r") hostname = f_hostname.read().rstrip() f_hostname.close() f_rel.close() if os.path.isfile(os.path.join(part_path, "var/log/dmesg")): f_dmesg = open( os.path.join(part_path, "var/log/dmesg"), "r") for linea in f_dmesg: aux = re.search(r"(Linux version [^\s]*)", linea) if aux: kernel_v = aux.group(1) break f_dmesg.close() out_f.write( "Distribution ID:\t\t{}\nDistribution Release:\t\t{}\nDistribution codename:\t\t{}\nDistribution description:\t{}\nKernel version:\t{}\nHostname:\t{}\n" .format(dist_id, dist_rel, dist_coden, dist_desc, kernel_v, hostname)) install_date = "" if os.path.isdir( os.path.join(self.myconfig('mountdir'), "p%s" % p.partition, "root")): item = os.path.join(self.myconfig('source'), 'mnt', "p%s" % p.partition, "root") install_date = self.filesystem.get_macb([item])[item][3] for f in [ "root/install.log", "var/log/installer/syslog", "root/anaconda-ks.cfg" ]: if os.path.isfile( os.path.join(self.myconfig('mountdir'), "p%s" % p.partition, f)): item = os.path.join(self.myconfig('source'), 'mnt', "p%s" % p.partition, f) install_date = self.filesystem.get_macb([item ])[item][3] break if install_date != "": out_f.write("Install date:\t{}\n\n".format(install_date)) # usuarios self.get_linux_lastlog(p.partition) temp = self.get_linux_wtmp(os.path.join(part_path, "var/log")) # temp = subprocess.check_output('last -f {} --time-format iso'.format(os.path.join(part_path, "var/log/wtmp")), shell=True).decode("utf-8") with open(self.outfile, 'a') as out_f: out_f.write("\nLogins:\n\n{}".format(temp))
def get_hive_files(self, path): """ Retrieves all hives found in source if path is not specified. Attrs: path: path to registry hive """ if path: if os.path.exists(path): return path else: raise base.job.RVTError('path {} does not exist'.format(path)) check_directory(self.myconfig('mountdir'), error_missing=True) regfiles = {} Find = GetFiles(self.config, vss=self.myflag("vss")) for main_hive in ['SYSTEM', 'SOFTWARE', 'SAM', 'SECURITY']: for item in Find.search( "/Windows/System32/config/{}$".format(main_hive)): hive = item.split('/')[-1].lower() if hive not in regfiles: # Get only the first hit regfiles[hive] = os.path.join(self.myconfig('casedir'), item) if "software" not in regfiles.keys(): self.logger().warning('No SOFTWARE hive found in source') return {} NTUSER = Find.search( r"/(Documents and settings|users)/.*/(NTUSER|UsrClass)\.dat$") usr = defaultdict(list) regfiles["ntuser"] = {} regfiles["usrclass"] = {} for item in NTUSER: aux = re.search("(Documents and settings|Users)/([^/]*)/", item, re.I) user = aux.group(2) hive_name = 'ntuser' if item.lower().endswith( "ntuser.dat") else 'usrclass' if user not in usr[hive_name]: usr[hive_name].append(user) else: # Get only the first hit continue if hive_name == "ntuser": regfiles["ntuser"][user] = os.path.join( self.myconfig('casedir'), item) else: regfiles["usrclass"][user] = os.path.join( self.myconfig('casedir'), item) amcache = list(Find.search("/Windows/AppCompat/Programs/Amcache.hve")) if len(amcache) != 0: regfiles["amcache"] = os.path.join(self.myconfig('casedir'), amcache[0]) syscache = list(Find.search(r"/syscache.hve$")) if len(syscache) != 0: regfiles["syscache"] = os.path.join(self.myconfig('casedir'), syscache[0]) return regfiles
def characterize_Windows(self): """ Characterize Windows partitions from registry files and timeline. """ hives_dir = self.myconfig('hivesdir') # Check registry is parsed. Generate the minimum files needed otherwise ripplugins_file = self.myconfig('ripplugins') if not check_directory(hives_dir): module = base.job.load_module( self.config, 'plugins.windows.RVT_autorip.Autorip', extra_config=dict(ripplugins=ripplugins_file)) list(module.run()) # Get the autorip outputfile associated with each necessary plugin with open(ripplugins_file) as rf: ripplugins = json.load(rf) used_plugins = [ 'winnt_cv', 'shutdown', 'timezone', 'lastloggedon', 'processor_architecture', 'compname', 'samparse', 'profilelist' ] os_plugins = [ 'winnt_cv', 'shutdown', 'timezone', 'lastloggedon', 'processor_architecture', 'compname' ] plugin_files = { plug: p['file'] for plug in used_plugins for p in ripplugins if plug in p['plugins'] } plugin_fields = { 'winnt_cv': [ ' ProductName', ' CurrentVersion', ' InstallationType', ' EditionID', ' CurrentBuild', ' ProductId', ' RegisteredOwner', ' RegisteredOrganization', ' InstallDate' ], 'shutdown': [' ShutdownTime'], 'processor_architecture': ['PROCESSOR_ARCHITECTURE'], 'compname': ['ComputerName'] } field_names = { ' ProductName': 'ProductName', ' CurrentVersion': 'CurrentVersion', ' InstallationType': 'InstallationType', ' EditionID': 'EditionID', ' CurrentBuild': 'CurrentBuild', ' ProductId': 'ProductId', ' RegisteredOwner': 'RegisteredOwner', ' RegisteredOrganization': 'RegisteredOrganization', ' InstallDate': 'InstallDate', ' ShutdownTime': 'ShutdownTime', ' TimeZoneKeyName': 'TimeZone', 'PROCESSOR_ARCHITECTURE': 'ProcessorArchitecture', 'ComputerName': 'ComputerName' } partitions = [ folder for folder in sorted(os.listdir(self.myconfig('mountdir'))) if folder.startswith('p') ] # Define self.ntusers, that gets the creation date of NTUSER.DAT for every user and partition self.make_ntuser_timeline() # Main loop to populate os_info os_info = defaultdict(dict) for part in partitions: for plug in os_plugins: hivefile = os.path.join( hives_dir, '{}_{}.txt'.format(plugin_files[plug], part)) if not check_file(hivefile): continue with open(hivefile) as f_in: if plug == 'lastloggedon': for line in f_in: if line.startswith('LastLoggedOn'): f_in.readline() last_write = f_in.readline()[11:].rstrip('\n') f_in.readline() last_user = f_in.readline()[22:].rstrip('\n') os_info[part][ 'LastLoggedOn'] = '{} ({})'.format( last_write, last_user) break continue elif plug == 'timezone': for line in f_in: if line.startswith('TimeZoneInformation'): bias, tz_name = '', '' while not line.startswith( '....................') and line != "": line = f_in.readline() if line.startswith(' Bias'): bias = line[line.find('('):].rstrip( '\n') if line.startswith(' TimeZoneKeyName'): line = line[len(' TimeZoneKeyName') + 3:].rstrip('\n') tz_name = line[:line.find('Time') + 4] os_info[part]['TimeZone'] = '{} {}'.format( tz_name, bias) break continue for field in plugin_fields[plug]: f_in.seek(0) for line in f_in: if line.startswith(field): os_info[part][ field_names[field]] = line[len(field) + 3:].rstrip('\n') break # Skip displaying partition info if it does not contain an OS if not os_info.get(part, None): self.logger().debug( 'No OS information for partition {}'.format(part)) continue # Users Info hivefile = os.path.join( hives_dir, '{}_{}.txt'.format(plugin_files['samparse'], part)) line = ' ' users = [] user_profiles = [] if check_file(hivefile): with open(hivefile) as f_in: # Parse samparse while not line.startswith('profilelist') and line != "": line = f_in.readline() aux = re.search(r"Username\s*:\s*(.*)\n", line) if aux: user = [aux.group(1), "", ""] while line != "\n": line = f_in.readline() aux = re.search( r"Account Created\s*:\s*(.*)\n", line) if aux: aux1 = aux.group(1).replace(" ", " ") date = datetime.datetime.strptime( aux1, '%a %b %d %H:%M:%S %Y Z') user[1] = date.strftime( '%d-%m-%Y %H:%M:%S UTC') continue aux = re.search( r"Last Login Date\s*:\s*(.*)\n", line) # TODO: check this field is reliable if aux: if aux.group(1).find("Never") == -1: aux1 = aux.group(1).replace(" ", " ") date = datetime.datetime.strptime( aux1, '%a %b %d %H:%M:%S %Y Z') user[2] = date.strftime( '%d-%m-%Y %H:%M:%S UTC') else: user[2] = "Never" users.append(user) break # Parse profilelist line = ' ' while not line.startswith( '....................') and line != "": line = f_in.readline() aux = re.match(r"Path\s*:\s*.:.Users.(.*)", line.strip()) if aux: # import pudb; pudb.set_trace() user = [aux.group(1), "", ""] while line != "\n": line = f_in.readline() aux = re.search(r"LastWrite\s*:\s*(.*)", line.strip()) if aux: aux1 = aux.group(1).replace(" ", " ") date = datetime.datetime.strptime( aux1, '%a %b %d %H:%M:%S %Y (UTC)') user[2] = date.strftime( "%d-%m-%Y %H:%M:%S UTC") user_profiles.append(user) # Get creation date from NTUSER.DAT if not found in profilelist for i in user_profiles: for j in self.ntusers[part]: if i[0] == j[0] and i[1] == "": i[1] = j[1].strftime('%d-%m-%Y %H:%M:%S UTC') os_info[part]["users"] = users os_info[part]["user_profiles"] = user_profiles return os_info
def run(self, path=""): self.disk = getSourceImage(self.myconfig) if not self.disk.exists(): self.logger().error(self.disk) return self.source = self.myconfig('source') self.outFolder = self.myconfig('deleteddir') check_directory(self.outFolder, create=True) # Set maximal dates for later update self.firstDate = datetime.date.today() + datetime.timedelta(days=365) self.lastDate = datetime.date(1970, 1, 1) # Process Timeline deleted files self.timelineBodyFile = os.path.join(self.myconfig('timelinesdir'), '{}_BODY.csv'.format(self.source)) check_file(self.timelineBodyFile, error_missing=True) # cmd = r"grep '(deleted' {} | grep -v FILE_NAME | cut -d'|' -f2 | sed 's_^[0-9-][0-9-]*/mnt/\(.*\) (deleted.*$_\1_' | sort -u".format(self.timelineBodyFile) cmd = r"grep '(deleted' {} | grep -v '\$FILE_NAME' | cut -d'|' -f2,3,7".format(self.timelineBodyFile) deletedTimelineFiles = shell_command(cmd) df_timeline = self.get_dataframe(deletedTimelineFiles, 'timeline') # Process Recycle self.recycleFile = os.path.join(self.myconfig('recycledir'), 'recycle_bin.csv') check_file(self.recycleFile, error_missing=True) df_recycle = self.get_dataframe(self.recycleFile, 'recycle') # Process UsnJrnl and INDX df_usnjrnl = pd.DataFrame() df_indx = pd.DataFrame() for p in self.disk.partitions: self.partName = ''.join(['p', p.partition]) if p.isMountable: self.usnJrnlFile = os.path.join(self.myconfig('journaldir'), 'UsnJrnl_{}.csv'.format(p.partition)) check_file(self.usnJrnlFile, error_missing=True) df_u = self.get_dataframe(shell_command(r"grep 'DELETE CLOSE' {} | cut -d',' -f 1,2,4".format(self.usnJrnlFile)), 'usnjrnl') self.indxFile = os.path.join(self.myconfig('timelinesdir'), '{}_INDX_timeline.csv'.format(p.partition)) if not check_file(self.indxFile): df_i = pd.DataFrame() # cmd = "grep -v 'SHORT FILENAME FORMAT' {} | grep -v 'NOT OBTAINED' | grep -v 'invalid MFTReference' | cut -d ';' -f 3,4,5,7".format(self.indxFile) # real # cmd = r"tail -n +2 {} | grep -va 'SHORT FILENAME FORMAT' | grep -va 'NOT OBTAINED' | grep -va 'invalid MFTReference' | cut -d ';' -f 2,5,9,14 ".format(self.indxFile) # unsorted # cmd = r"tail -n +2 {} | grep -va 'SHORT FILENAME FORMAT' | grep -va 'NOT OBTAINED' | cut -d ';' -f 2,5,9,14 ".format(self.indxFile) # unsorted cmd = r"tail -n +2 {} | grep -va 'SHORT FILENAME FORMAT' | grep -va 'NOT OBTAINED' | cut -d ';' -f 3,4,6,7,9 ".format(self.indxFile) # real df_i = self.get_dataframe(shell_command(cmd), 'indx') df_usnjrnl = self.join_dataframes(df_usnjrnl, df_u) df_indx = self.join_dataframes(df_indx, df_i) # TODO: timeline_all does not need columns source or reliable # Compare Timeline against INDX to extract unique (assuming deleted) files in INDX cmd = r"cut -d'|' -f2 {} | grep -v '\$FILE_NAME'".format(self.timelineBodyFile) df_all_timeline = self.get_dataframe(shell_command(cmd), 'timeline_all') self.logger().debug('Obtaining unique files in INDX') df_indx = self.get_deleted_in_INDX(df_all_timeline, df_indx) # Create a global dataframe with all artifacts self.logger().info('Combining artifacts to create a full list of deleted files') df_global = self.combine_artifacts([df_usnjrnl, df_recycle, df_timeline, df_indx]) print(df_global.shape, df_global.columns) duplicated_bin = df_global.duplicated('Filename', keep='first') # First sources have precedence self.logger().info('Found {} duplicated files merging sources'.format(duplicated_bin.sum())) print('before dropping', df_global.shape) df_global = df_global[~duplicated_bin] # df_global.drop_duplicates('Filename', keep='first', inplace=True) print('after dropping', df_global.shape) print(df_global.columns) print(df_global.head()) # Save global DataFrame # outfile = os.path.join(self.outFolder, '{}_deleted.csv'.format(self.source)) outfile = '/home/pgarcia/global_deleted.csv' with open(outfile, 'w') as f: f.write(df_global.to_csv(index=False)) # exit() # Create number of files summary based on day, hour and partition self.get_stats(self.join_dataframes(df_usnjrnl, df_recycle), 'all')