def process_yara_options(cls, config: Dict[str, Any]): rules = None if config.get('yara_rules', None) is not None: rule = config['yara_rules'] if rule[0] not in ["{", "/"]: rule = f'"{rule}"' if config.get('case', False): rule += " nocase" if config.get('wide', False): rule += " wide ascii" rules = yara.compile( sources={ 'n': f'rule r1 {{strings: $a = {rule} condition: $a}}' }) elif config.get('yara_source', None) is not None: rules = yara.compile(source=config['yara_source']) elif config.get('yara_file', None) is not None: rules = yara.compile(file=resources.ResourceAccessor().open( config['yara_file'], "rb")) elif config.get('yara_compiled_file', None) is not None: rules = yara.load(file=resources.ResourceAccessor().open( config['yara_compiled_file'], "rb")) else: vollog.error("No yara rules, nor yara rules file were specified") return rules
def stack( cls, context: interfaces.context.ContextInterface, layer_name: str, progress_callback: constants.ProgressCallback = None ) -> Optional[interfaces.layers.DataLayerInterface]: """Attempt to stack this based on the starting information.""" memlayer = context.layers[layer_name] if not isinstance(memlayer, physical.FileLayer): return None location = memlayer.location if location.endswith(".vmem"): vmss = location[:-5] + ".vmss" vmsn = location[:-5] + ".vmsn" current_layer_name = context.layers.free_layer_name( "VmwareMetaLayer") current_config_path = interfaces.configuration.path_join( "automagic", "layer_stacker", "stack", current_layer_name) vmss_success = False try: _ = resources.ResourceAccessor().open(vmss).read(10) context.config[interfaces.configuration.path_join( current_config_path, "location")] = vmss context.layers.add_layer( physical.FileLayer(context, current_config_path, current_layer_name)) vmss_success = True except IOError: pass vmsn_success = False if not vmss_success: try: _ = resources.ResourceAccessor().open(vmsn).read(10) context.config[interfaces.configuration.path_join( current_config_path, "location")] = vmsn context.layers.add_layer( physical.FileLayer(context, current_config_path, current_layer_name)) vmsn_success = True except IOError: pass vollog.log( constants.LOGLEVEL_VVVV, f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})" ) if not vmss_success and not vmsn_success: return None new_layer_name = context.layers.free_layer_name("VmwareLayer") context.config[interfaces.configuration.path_join( current_config_path, "base_layer")] = layer_name context.config[interfaces.configuration.path_join( current_config_path, "meta_layer")] = current_layer_name new_layer = VmwareLayer(context, current_config_path, new_layer_name) return new_layer return None
def retreive_pdb( self, guid: str, file_name: str, progress_callback: constants.ProgressCallback = None ) -> Optional[str]: vollog.info("Download PDB file...") file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) for sym_url in ['http://msdl.microsoft.com/download/symbols']: url = sym_url + "/{}/{}/".format(file_name, guid) result = None for suffix in [file_name, file_name[:-1] + '_']: try: vollog.debug("Attempting to retrieve {}".format(url + suffix)) # We have to cache this because the file is opened by a layer and we can't control whether that caches result = resources.ResourceAccessor( progress_callback).open(url + suffix) except (error.HTTPError, error.URLError) as excp: vollog.debug("Failed with {}".format(excp)) if result: break if progress_callback is not None: progress_callback(100, "Downloading {}".format(url + suffix)) if result is None: return None return url + suffix
def __init__(self, location: str): self._location = location with resources.ResourceAccessor().open(url=location) as fp: self._data = json.load(fp) if not self._verify(): raise ValueError( "Unsupported version for remote banner list format")
def retreive_pdb( self, guid: str, file_name: str, progress_callback: constants.ProgressCallback = None ) -> Optional[str]: vollog.info("Download PDB file...") file_name = ".".join(file_name.split(".")[:-1] + ['pdb']) for sym_url in ['http://msdl.microsoft.com/download/symbols']: url = sym_url + "/{}/{}/".format(file_name, guid) result = None for suffix in [file_name, file_name[:-1] + '_']: try: vollog.debug("Attempting to retrieve {}".format(url + suffix)) # We no longer cache it, so this is a glorified remote endpoint check result = resources.ResourceAccessor( progress_callback, enable_cache=False).open(url + suffix) except (error.HTTPError, error.URLError) as excp: vollog.debug("Failed with {}".format(excp)) if result: break if progress_callback is not None: progress_callback(100, "Downloading {}".format(url + suffix)) if result is None: return None return url + suffix
def _generator(self) -> Generator[Tuple, None, None]: """Generates results from a strings file.""" revmap = self.generate_mapping(self.config['primary']) accessor = resources.ResourceAccessor() strings_fp = accessor.open(self.config['strings_file'], "rb") strings_size = path.getsize(strings_fp.file.name) line = strings_fp.readline() last_prog = 0 while line: try: offset, string = self._parse_line(line) try: revmap_list = [ name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12] ] except (IndexError, KeyError): revmap_list = ["FREE MEMORY"] yield (0, (str(string, 'latin-1'), format_hints.Hex(offset), ", ".join(revmap_list))) except ValueError: vollog.error("Strings file is in the wrong format") return line = strings_fp.readline() prog = strings_fp.tell() / strings_size * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) self._progress_callback(prog, "Matching strings in memory")
def run_script(self, location: str): """Runs a python script within the context of volshell""" if not parse.urlparse(location).scheme: location = "file:" + request.pathname2url(location) print(f"Running code from {location}\n") accessor = resources.ResourceAccessor() with io.TextIOWrapper(accessor.open(url = location), encoding = 'utf-8') as fp: self.__console.runsource(fp.read(), symbol = 'exec') print("\nCode complete")
def _generator(self): if self.config.get('isf', None) is not None: file_list = [self.config['isf']] else: file_list = list(self.list_all_isf_files()) # Filter the files filtered_list = [] if not len(self.config['filter']): filtered_list = file_list else: for isf_file in file_list: for filter_item in self.config['filter']: if filter_item in isf_file: filtered_list.append(isf_file) try: import jsonschema if not self.config['validate']: raise ImportError # Act as if we couldn't import if validation is turned off def check_valid(data): return "True" if schemas.validate(data, True) else "False" except ImportError: def check_valid(data): return "Unknown" # Process the filtered list for entry in filtered_list: num_types = num_enums = num_bases = num_symbols = 0 windows_info = linux_banner = mac_banner = renderers.NotAvailableValue( ) valid = "Unknown" with resources.ResourceAccessor().open(url=entry) as fp: try: data = json.load(fp) num_symbols = len(data.get('symbols', [])) num_types = len(data.get('user_types', [])) num_enums = len(data.get('enums', [])) num_bases = len(data.get('base_types', [])) linux_banner = self._get_banner(linux.LinuxBannerCache, data) mac_banner = self._get_banner(mac.MacBannerCache, data) if not linux_banner and not mac_banner: windows_info = os.path.splitext( os.path.basename(entry))[0] valid = check_valid(data) except (UnicodeDecodeError, json.decoder.JSONDecodeError): vollog.warning("Invalid ISF: {}".format(entry)) yield (0, (entry, valid, num_bases, num_types, num_symbols, num_enums, windows_info, linux_banner, mac_banner))
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str, metadata: Optional[Dict[str, Any]] = None) -> None: super().__init__(context = context, config_path = config_path, name = name, metadata = metadata) self._write_warning = False self._location = self.config["location"] self._accessor = resources.ResourceAccessor() self._file_: Optional[IO[Any]] = None self._size: Optional[int] = None # Construct the lock now (shared if made before threading) in case we ever need it self._lock: Union[DummyLock, threading.Lock] = DummyLock() if constants.PARALLELISM == constants.Parallelism.Threading: self._lock = threading.Lock() # Instantiate the file to throw exceptions if the file doesn't open _ = self._file
def _generator(self) -> Generator[Tuple, None, None]: """Generates results from a strings file.""" string_list: List[Tuple[int, bytes]] = [] # Test strings file format is accurate accessor = resources.ResourceAccessor() strings_fp = accessor.open(self.config['strings_file'], "rb") line = strings_fp.readline() count: float = 0 while line: count += 1 try: offset, string = self._parse_line(line) string_list.append((offset, string)) except ValueError: vollog.error(f"Line in unrecognized format: line {count}") line = strings_fp.readline() kernel = self.context.modules[self.config['kernel']] revmap = self.generate_mapping( self.context, kernel.layer_name, kernel.symbol_table_name, progress_callback=self._progress_callback, pid_list=self.config['pid']) last_prog: float = 0 line_count: float = 0 num_strings = len(string_list) for offset, string in string_list: line_count += 1 try: revmap_list = [ name + ":" + hex(offset) for (name, offset) in revmap[offset >> 12] ] except (IndexError, KeyError): revmap_list = ["FREE MEMORY"] yield (0, (str(string, 'latin-1'), format_hints.Hex(offset), ", ".join(revmap_list))) prog = line_count / num_strings * 100 if round(prog, 1) > last_prog: last_prog = round(prog, 1) self._progress_callback(prog, "Matching strings in memory")
def __init__(self, context: interfaces.context.ContextInterface, config_path: str, name: str, isf_url: str, native_types: interfaces.symbols.NativeTableInterface = None, table_mapping: Optional[Dict[str, str]] = None, validate: bool = True, class_types: Optional[Mapping[ str, Type[interfaces.objects.ObjectInterface]]] = None, symbol_mask: int = 0) -> None: """Instantiates a SymbolTable based on an IntermediateSymbolFormat JSON file. This is validated against the appropriate schema. The validation can be disabled by passing validate = False, but this should almost never be done. Args: context: The volatility context for the symbol table config_path: The configuration path for the symbol table name: The name for the symbol table (this is used in symbols e.g. table!symbol ) isf_url: The URL pointing to the ISF file location native_types: The NativeSymbolTable that contains the native types for this symbol table table_mapping: A dictionary linking names referenced in the file with symbol tables in the context validate: Determines whether the ISF file will be validated against the appropriate schema class_types: A dictionary of type names and classes that override StructType when they are instantiated symbol_mask: An address mask used for all returned symbol offsets from this table (a mask of 0 disables masking) """ # Check there are no obvious errors # Open the file and test the version self._versions = dict([(x.version, x) for x in class_subclasses(ISFormatTable)]) fp = resources.ResourceAccessor().open(isf_url) reader = codecs.getreader("utf-8") json_object = json.load(reader(fp)) # type: ignore fp.close() # Validation is expensive, but we cache to store the hashes of successfully validated json objects if validate and not schemas.validate(json_object): raise exceptions.SymbolSpaceError( f"File does not pass version validation: {isf_url}") metadata = json_object.get('metadata', None) # Determine the delegate or throw an exception self._delegate = self._closest_version(metadata.get( 'format', "0.0.0"), self._versions)(context, config_path, name, json_object, native_types, table_mapping) if self._delegate.version < constants.ISF_MINIMUM_SUPPORTED: raise RuntimeError( "ISF version {} is no longer supported: {}".format( metadata.get('format', "0.0.0"), isf_url)) elif self._delegate.version < constants.ISF_MINIMUM_DEPRECATED: vollog.warning( f"ISF version {metadata.get('format', '0.0.0')} has been deprecated: {isf_url}" ) # Inherit super().__init__(context, config_path, name, native_types or self._delegate.natives, table_mapping=table_mapping, class_types=class_types) # Since we've been created with parameters, ensure our config is populated likewise self.config['isf_url'] = isf_url self.config['symbol_mask'] = symbol_mask
def _generator(self): if not has_yara: log.error("You must install yara") return config = dict() if self.config.get('yara_file', None) is not None: RULES = yara.compile(file=resources.ResourceAccessor().open( self.config['yara_file'], "rb")) else: # https://github.com/Yara-Rules/rules/blob/master/malware/MALW_Pony.yar SIGS = { 'pony': ''' rule pony { meta: author = "Brian Wallace @botnet_hunter" author_email = "*****@*****.**" date = "2014-08-16" description = "Identify Pony" strings: $ = "YUIPWDFILE0YUIPKDFILE0YUICRYPTED0YUI1.0" condition: all of them } ''' } RULES = yara.compile(sources=SIGS) #filter_func = pslist.PsList.create_pid_filter([self.config.get('pid', None)]) filter_func = pslist.PsList.create_pid_filter( [self.config.get('pid', None)]) for task in pslist.PsList.list_processes( context=self.context, layer_name=self.config['primary'], symbol_table=self.config['nt_symbols'], filter_func=filter_func): try: proc_layer_name = task.add_process_layer() except exceptions.InvalidAddressException: continue proc_layer = self.context.layers[proc_layer_name] for offset, rule_name, name, value in proc_layer.scan( context=self.context, scanner=yarascan.YaraScanner(rules=RULES), sections=vadyarascan.VadYaraScan.get_vad_maps(task)): log.debug("Got a Yara match!") vad, vad_start, vad_end = self.get_vad(task, offset) if vad is None: log.debug("VAD not found") return full_pe = io.BytesIO() chunk_size = 1024 * 1024 * 10 #vadinfo.VadInfo.vad_dump(self.context, task, vad, full_pe) offset = vad_start while offset < vad_end: to_read = min(chunk_size, vad_end - offset) data = proc_layer.read(offset, to_read, pad=True) if not data: break full_pe.write(data) offset += to_read if not full_pe: continue config = self.get_config(full_pe.getvalue()) if not config: log.debug("Config extraction failed") continue yield (0, (format_hints.Hex(offset), task.UniqueProcessId, str(config)))