def process_yara_options(cls, config: Dict[str, Any]):
rules = None
if config.get('yara_rules', None) is not None:
rule = config['yara_rules']
if rule[0] not in ["{", "/"]:
rule = f'"{rule}"'
if config.get('case', False):
rule += " nocase"
if config.get('wide', False):
rule += " wide ascii"
rules = yara.compile(
sources={
'n': f'rule r1 {{strings: $a = {rule} condition: $a}}'
})
elif config.get('yara_source', None) is not None:
rules = yara.compile(source=config['yara_source'])
elif config.get('yara_file', None) is not None:
rules = yara.compile(file=resources.ResourceAccessor().open(
config['yara_file'], "rb"))
elif config.get('yara_compiled_file', None) is not None:
rules = yara.load(file=resources.ResourceAccessor().open(
config['yara_compiled_file'], "rb"))
else:
vollog.error("No yara rules, nor yara rules file were specified")
return rules
def stack(
cls,
context: interfaces.context.ContextInterface,
layer_name: str,
progress_callback: constants.ProgressCallback = None
) -> Optional[interfaces.layers.DataLayerInterface]:
"""Attempt to stack this based on the starting information."""
memlayer = context.layers[layer_name]
if not isinstance(memlayer, physical.FileLayer):
return None
location = memlayer.location
if location.endswith(".vmem"):
vmss = location[:-5] + ".vmss"
vmsn = location[:-5] + ".vmsn"
current_layer_name = context.layers.free_layer_name(
"VmwareMetaLayer")
current_config_path = interfaces.configuration.path_join(
"automagic", "layer_stacker", "stack", current_layer_name)
vmss_success = False
try:
_ = resources.ResourceAccessor().open(vmss).read(10)
context.config[interfaces.configuration.path_join(
current_config_path, "location")] = vmss
context.layers.add_layer(
physical.FileLayer(context, current_config_path,
current_layer_name))
vmss_success = True
except IOError:
pass
vmsn_success = False
if not vmss_success:
try:
_ = resources.ResourceAccessor().open(vmsn).read(10)
context.config[interfaces.configuration.path_join(
current_config_path, "location")] = vmsn
context.layers.add_layer(
physical.FileLayer(context, current_config_path,
current_layer_name))
vmsn_success = True
except IOError:
pass
vollog.log(
constants.LOGLEVEL_VVVV,
f"Metadata found: VMSS ({vmss_success}) or VMSN ({vmsn_success})"
)
if not vmss_success and not vmsn_success:
return None
new_layer_name = context.layers.free_layer_name("VmwareLayer")
context.config[interfaces.configuration.path_join(
current_config_path, "base_layer")] = layer_name
context.config[interfaces.configuration.path_join(
current_config_path, "meta_layer")] = current_layer_name
new_layer = VmwareLayer(context, current_config_path,
new_layer_name)
return new_layer
return None
def retreive_pdb(
self,
guid: str,
file_name: str,
progress_callback: constants.ProgressCallback = None
) -> Optional[str]:
vollog.info("Download PDB file...")
file_name = ".".join(file_name.split(".")[:-1] + ['pdb'])
for sym_url in ['http://msdl.microsoft.com/download/symbols']:
url = sym_url + "/{}/{}/".format(file_name, guid)
result = None
for suffix in [file_name, file_name[:-1] + '_']:
try:
vollog.debug("Attempting to retrieve {}".format(url +
suffix))
# We have to cache this because the file is opened by a layer and we can't control whether that caches
result = resources.ResourceAccessor(
progress_callback).open(url + suffix)
except (error.HTTPError, error.URLError) as excp:
vollog.debug("Failed with {}".format(excp))
if result:
break
if progress_callback is not None:
progress_callback(100, "Downloading {}".format(url + suffix))
if result is None:
return None
return url + suffix
def __init__(self, location: str):
self._location = location
with resources.ResourceAccessor().open(url=location) as fp:
self._data = json.load(fp)
if not self._verify():
raise ValueError(
"Unsupported version for remote banner list format")
def retreive_pdb(
self,
guid: str,
file_name: str,
progress_callback: constants.ProgressCallback = None
) -> Optional[str]:
vollog.info("Download PDB file...")
file_name = ".".join(file_name.split(".")[:-1] + ['pdb'])
for sym_url in ['http://msdl.microsoft.com/download/symbols']:
url = sym_url + "/{}/{}/".format(file_name, guid)
result = None
for suffix in [file_name, file_name[:-1] + '_']:
try:
vollog.debug("Attempting to retrieve {}".format(url +
suffix))
# We no longer cache it, so this is a glorified remote endpoint check
result = resources.ResourceAccessor(
progress_callback,
enable_cache=False).open(url + suffix)
except (error.HTTPError, error.URLError) as excp:
vollog.debug("Failed with {}".format(excp))
if result:
break
if progress_callback is not None:
progress_callback(100, "Downloading {}".format(url + suffix))
if result is None:
return None
return url + suffix
def _generator(self) -> Generator[Tuple, None, None]:
"""Generates results from a strings file."""
revmap = self.generate_mapping(self.config['primary'])
accessor = resources.ResourceAccessor()
strings_fp = accessor.open(self.config['strings_file'], "rb")
strings_size = path.getsize(strings_fp.file.name)
line = strings_fp.readline()
last_prog = 0
while line:
try:
offset, string = self._parse_line(line)
try:
revmap_list = [
name + ":" + hex(offset)
for (name, offset) in revmap[offset >> 12]
]
except (IndexError, KeyError):
revmap_list = ["FREE MEMORY"]
yield (0, (str(string, 'latin-1'), format_hints.Hex(offset),
", ".join(revmap_list)))
except ValueError:
vollog.error("Strings file is in the wrong format")
return
line = strings_fp.readline()
prog = strings_fp.tell() / strings_size * 100
if round(prog, 1) > last_prog:
last_prog = round(prog, 1)
self._progress_callback(prog, "Matching strings in memory")
def run_script(self, location: str):
"""Runs a python script within the context of volshell"""
if not parse.urlparse(location).scheme:
location = "file:" + request.pathname2url(location)
print(f"Running code from {location}\n")
accessor = resources.ResourceAccessor()
with io.TextIOWrapper(accessor.open(url = location), encoding = 'utf-8') as fp:
self.__console.runsource(fp.read(), symbol = 'exec')
print("\nCode complete")
def _generator(self):
if self.config.get('isf', None) is not None:
file_list = [self.config['isf']]
else:
file_list = list(self.list_all_isf_files())
# Filter the files
filtered_list = []
if not len(self.config['filter']):
filtered_list = file_list
else:
for isf_file in file_list:
for filter_item in self.config['filter']:
if filter_item in isf_file:
filtered_list.append(isf_file)
try:
import jsonschema
if not self.config['validate']:
raise ImportError # Act as if we couldn't import if validation is turned off
def check_valid(data):
return "True" if schemas.validate(data, True) else "False"
except ImportError:
def check_valid(data):
return "Unknown"
# Process the filtered list
for entry in filtered_list:
num_types = num_enums = num_bases = num_symbols = 0
windows_info = linux_banner = mac_banner = renderers.NotAvailableValue(
)
valid = "Unknown"
with resources.ResourceAccessor().open(url=entry) as fp:
try:
data = json.load(fp)
num_symbols = len(data.get('symbols', []))
num_types = len(data.get('user_types', []))
num_enums = len(data.get('enums', []))
num_bases = len(data.get('base_types', []))
linux_banner = self._get_banner(linux.LinuxBannerCache,
data)
mac_banner = self._get_banner(mac.MacBannerCache, data)
if not linux_banner and not mac_banner:
windows_info = os.path.splitext(
os.path.basename(entry))[0]
valid = check_valid(data)
except (UnicodeDecodeError, json.decoder.JSONDecodeError):
vollog.warning("Invalid ISF: {}".format(entry))
yield (0, (entry, valid, num_bases, num_types, num_symbols,
num_enums, windows_info, linux_banner, mac_banner))
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
metadata: Optional[Dict[str, Any]] = None) -> None:
super().__init__(context = context, config_path = config_path, name = name, metadata = metadata)
self._write_warning = False
self._location = self.config["location"]
self._accessor = resources.ResourceAccessor()
self._file_: Optional[IO[Any]] = None
self._size: Optional[int] = None
# Construct the lock now (shared if made before threading) in case we ever need it
self._lock: Union[DummyLock, threading.Lock] = DummyLock()
if constants.PARALLELISM == constants.Parallelism.Threading:
self._lock = threading.Lock()
# Instantiate the file to throw exceptions if the file doesn't open
_ = self._file
def _generator(self) -> Generator[Tuple, None, None]:
"""Generates results from a strings file."""
string_list: List[Tuple[int, bytes]] = []
# Test strings file format is accurate
accessor = resources.ResourceAccessor()
strings_fp = accessor.open(self.config['strings_file'], "rb")
line = strings_fp.readline()
count: float = 0
while line:
count += 1
try:
offset, string = self._parse_line(line)
string_list.append((offset, string))
except ValueError:
vollog.error(f"Line in unrecognized format: line {count}")
line = strings_fp.readline()
kernel = self.context.modules[self.config['kernel']]
revmap = self.generate_mapping(
self.context,
kernel.layer_name,
kernel.symbol_table_name,
progress_callback=self._progress_callback,
pid_list=self.config['pid'])
last_prog: float = 0
line_count: float = 0
num_strings = len(string_list)
for offset, string in string_list:
line_count += 1
try:
revmap_list = [
name + ":" + hex(offset)
for (name, offset) in revmap[offset >> 12]
]
except (IndexError, KeyError):
revmap_list = ["FREE MEMORY"]
yield (0, (str(string, 'latin-1'), format_hints.Hex(offset),
", ".join(revmap_list)))
prog = line_count / num_strings * 100
if round(prog, 1) > last_prog:
last_prog = round(prog, 1)
self._progress_callback(prog, "Matching strings in memory")
def __init__(self,
context: interfaces.context.ContextInterface,
config_path: str,
name: str,
isf_url: str,
native_types: interfaces.symbols.NativeTableInterface = None,
table_mapping: Optional[Dict[str, str]] = None,
validate: bool = True,
class_types: Optional[Mapping[
str, Type[interfaces.objects.ObjectInterface]]] = None,
symbol_mask: int = 0) -> None:
"""Instantiates a SymbolTable based on an IntermediateSymbolFormat JSON file. This is validated against the
appropriate schema. The validation can be disabled by passing validate = False, but this should almost never be
done.
Args:
context: The volatility context for the symbol table
config_path: The configuration path for the symbol table
name: The name for the symbol table (this is used in symbols e.g. table!symbol )
isf_url: The URL pointing to the ISF file location
native_types: The NativeSymbolTable that contains the native types for this symbol table
table_mapping: A dictionary linking names referenced in the file with symbol tables in the context
validate: Determines whether the ISF file will be validated against the appropriate schema
class_types: A dictionary of type names and classes that override StructType when they are instantiated
symbol_mask: An address mask used for all returned symbol offsets from this table (a mask of 0 disables masking)
"""
# Check there are no obvious errors
# Open the file and test the version
self._versions = dict([(x.version, x)
for x in class_subclasses(ISFormatTable)])
fp = resources.ResourceAccessor().open(isf_url)
reader = codecs.getreader("utf-8")
json_object = json.load(reader(fp)) # type: ignore
fp.close()
# Validation is expensive, but we cache to store the hashes of successfully validated json objects
if validate and not schemas.validate(json_object):
raise exceptions.SymbolSpaceError(
f"File does not pass version validation: {isf_url}")
metadata = json_object.get('metadata', None)
# Determine the delegate or throw an exception
self._delegate = self._closest_version(metadata.get(
'format', "0.0.0"), self._versions)(context, config_path, name,
json_object, native_types,
table_mapping)
if self._delegate.version < constants.ISF_MINIMUM_SUPPORTED:
raise RuntimeError(
"ISF version {} is no longer supported: {}".format(
metadata.get('format', "0.0.0"), isf_url))
elif self._delegate.version < constants.ISF_MINIMUM_DEPRECATED:
vollog.warning(
f"ISF version {metadata.get('format', '0.0.0')} has been deprecated: {isf_url}"
)
# Inherit
super().__init__(context,
config_path,
name,
native_types or self._delegate.natives,
table_mapping=table_mapping,
class_types=class_types)
# Since we've been created with parameters, ensure our config is populated likewise
self.config['isf_url'] = isf_url
self.config['symbol_mask'] = symbol_mask
def _generator(self):
if not has_yara:
log.error("You must install yara")
return
config = dict()
if self.config.get('yara_file', None) is not None:
RULES = yara.compile(file=resources.ResourceAccessor().open(
self.config['yara_file'], "rb"))
else:
# https://github.com/Yara-Rules/rules/blob/master/malware/MALW_Pony.yar
SIGS = {
'pony':
'''
rule pony {
meta:
author = "Brian Wallace @botnet_hunter"
author_email = "*****@*****.**"
date = "2014-08-16"
description = "Identify Pony"
strings:
$ = "YUIPWDFILE0YUIPKDFILE0YUICRYPTED0YUI1.0"
condition:
all of them
}
'''
}
RULES = yara.compile(sources=SIGS)
#filter_func = pslist.PsList.create_pid_filter([self.config.get('pid', None)])
filter_func = pslist.PsList.create_pid_filter(
[self.config.get('pid', None)])
for task in pslist.PsList.list_processes(
context=self.context,
layer_name=self.config['primary'],
symbol_table=self.config['nt_symbols'],
filter_func=filter_func):
try:
proc_layer_name = task.add_process_layer()
except exceptions.InvalidAddressException:
continue
proc_layer = self.context.layers[proc_layer_name]
for offset, rule_name, name, value in proc_layer.scan(
context=self.context,
scanner=yarascan.YaraScanner(rules=RULES),
sections=vadyarascan.VadYaraScan.get_vad_maps(task)):
log.debug("Got a Yara match!")
vad, vad_start, vad_end = self.get_vad(task, offset)
if vad is None:
log.debug("VAD not found")
return
full_pe = io.BytesIO()
chunk_size = 1024 * 1024 * 10
#vadinfo.VadInfo.vad_dump(self.context, task, vad, full_pe)
offset = vad_start
while offset < vad_end:
to_read = min(chunk_size, vad_end - offset)
data = proc_layer.read(offset, to_read, pad=True)
if not data:
break
full_pe.write(data)
offset += to_read
if not full_pe:
continue
config = self.get_config(full_pe.getvalue())
if not config:
log.debug("Config extraction failed")
continue
yield (0, (format_hints.Hex(offset), task.UniqueProcessId,
str(config)))
