def render(self, renderer):
result = self.parse_pdb()
if self.plugin_args.output_filename:
with renderer.open(filename=self.plugin_args.output_filename,
directory=self.plugin_args.dump_dir,
mode="w") as fd:
fd.write(utils.PPrint(result))
else:
renderer.write(utils.PPrint(result))
def render(self, renderer):
if self.plugin_args.converter:
cls = ProfileConverter.classes.get(self.plugin_args.converter)
if not cls:
raise IOError("Unknown converter %s" %
self.plugin_args.converter)
return cls(self.plugin_args.source,
profile_class=self.plugin_args.profile_class).Convert()
try:
input = io_manager.Factory(self.plugin_args.source,
session=self.session,
mode="r")
except IOError:
self.session.logging.critical(
"Input profile file %s could not be opened.",
self.plugin_args.source)
return
with input:
profile = self.ConvertProfile(input)
if profile:
with renderer.open(filename=self.plugin_args.out_file,
mode="wt") as output:
output.write(utils.PPrint(profile))
self.session.logging.info("Converted %s to %s", input,
output.name)
def Build(self, renderer):
repository = self.args.repository
changed_files = False
for source in self.args.sources:
profile_name = "OSX/%s" % source.split("/")[-1]
profile_metadata = repository.Metadata(profile_name)
# Profile does not exist - rebuild it.
if not profile_metadata:
data = repository.GetData(source)
# Transform the data as required.
data = self.TransformProfile(data)
repository.StoreData(profile_name,
utils.PPrint(data),
raw=True)
renderer.format("Building profile {0} from {1}\n",
profile_name, source)
changed_files = True
if changed_files and self.args.index or self.args.force_build_index:
renderer.format("Building index for profile {0} from {1}\n",
self.args.profile_name, self.args.index)
self.BuildIndex()
def Build(self, renderer):
repository = self.args.repository
profile_metadata = repository.Metadata(self.args.profile_name)
sources = []
for pattern in self.args.patterns:
sources.extend(fnmatch.filter(repository.ListFiles(), pattern))
# Find the latest modified source
last_modified = 0
for source in sources:
source_metadata = repository.Metadata(source)
last_modified = max(last_modified, source_metadata["LastModified"])
if not profile_metadata or (last_modified >
profile_metadata["LastModified"]):
definitions = []
for source in sources:
definitions.extend(
yaml.safe_load_all(repository.GetData(source, raw=True)))
# Transform the data as required.
data = {
"$ARTIFACTS": definitions,
"$METADATA": dict(ProfileClass="ArtifactProfile", )
}
repository.StoreData(self.args.profile_name,
utils.PPrint(data),
raw=True)
renderer.format("Building artifact profile {0}\n",
self.args.profile_name)
def Encoder(self, data, **options):
if options.get("raw"):
return utils.SmartStr(data)
if self.pretty_print:
return utils.PPrint(data)
return json.dumps(data, sort_keys=True, **options)
def Encoder(self, data, **options):
if options.get("raw"):
return utils.SmartStr(data)
# If the user specifically wants to encode in yaml, then do so.
if options.get("yaml"):
return yaml.safe_dump(data, default_flow_style=False)
return utils.PPrint(data)
def render(self, renderer):
vtypes = self.parser.VType()
result = {
"$METADATA": dict(
Type="Profile",
# This should probably be changed for something more specific.
ProfileClass=self.plugin_args.profile_class),
"$STRUCTS": vtypes,
"$ENUMS": vtypes.pop("$ENUMS", {}),
"$REVENUMS": vtypes.pop("$REVENUMS", {}),
}
renderer.write(utils.PPrint(result))
def Build(self, renderer):
repository = self.args.repository
profile_metadata = repository.Metadata(self.args.profile_name)
source_metadata = repository.Metadata(self.args.source)
if not profile_metadata or (source_metadata["LastModified"] >
profile_metadata["LastModified"]):
data = repository.GetData(self.args.source)
# Transform the data as required.
data = self.TransformProfile(data)
repository.StoreData(self.args.profile_name,
utils.PPrint(data),
raw=True)
renderer.format("Building profile {0} from {1}\n",
self.args.profile_name, self.args.source)
def BuildAll(self, renderer):
repository = self.args.repository
guid_file = self.args.repository.GetData(self.args.guids)
rejects_filename = self.args.guids + ".rejects"
rejects = self.args.repository.GetData(rejects_filename, default={})
reject_len = len(rejects)
try:
changed_files = set()
for pdb_filename, guids in guid_file.iteritems():
for guid in guids:
if guid in rejects:
continue
# If the profile exists in the repository continue.
if repository.Metadata("%s/%s" %
(self.args.profile_name, guid)):
continue
def Reject(e, guid=guid, changed_files=changed_files):
print "GUID %s rejected: %s" % (guid, e)
rejects[guid] = str(e)
changed_files.remove(guid)
# Otherwise build it.
changed_files.add(guid)
self.pool.AddTask(self.LaunchBuilder,
("build", "%s/%s" %
(pdb_filename, guid)),
on_error=Reject)
self.pool.Stop()
if changed_files and self.args.index or self.args.force_build_index:
renderer.format("Building index for profile {0} from {1}\n",
self.args.profile_name, self.args.index)
self.BuildIndex()
finally:
if len(rejects) != reject_len:
repository.StoreData(rejects_filename,
utils.PPrint(rejects),
raw=True)
renderer.format("Updating inventory.\n")
repository.StoreData("inventory", repository.RebuildInventory())
def fetch_and_parse(self, module_name=None, guid=None, renderer=None):
if module_name is None:
module_name = self.module_name
if guid is None:
guid = self.guid
# Allow the user to specify the required profile by name.
m = re.match("([^/]+)/GUID/([^/]+)$", module_name)
if m:
module_name = m.group(1)
guid = m.group(2)
if not guid or not module_name:
raise TypeError("GUID not specified.")
profile_name = "{0}/GUID/{1}".format(module_name.lower(), guid)
# Get the first repository to write to.
repository = self.session.repository_managers[0][1]
if module_name != "nt":
data = self._fetch_and_parse(module_name, guid)
if self.dumpfile:
with renderer.open(filename=self.dumpfile, mode="wt") as fd:
fd.write(utils.PPrint(data))
return repository.StoreData(profile_name, data)
for module_name in common.KERNEL_NAMES:
if module_name.endswith(".pdb"):
module_name, _ = os.path.splitext(module_name)
try:
data = self._fetch_and_parse(module_name, guid)
self.session.logging.warning(
"Profile %s fetched and built. Please "
"consider reporting this profile to the "
"Rekall team so we may add it to the public "
"profile repository.", profile_name)
return repository.StoreData(profile_name, data)
except IOError as e:
self.session.logging.error("Error: %s", e)
raise IOError("Profile not found")
def run(self, key, cb, *args, **kwargs):
raw = kwargs.pop("raw", False)
desc = kwargs.pop("desc", None)
if not self.cache_dir:
now = time.time()
result = cb(*args)
if desc:
logging.info("Completed %s in %d Seconds", desc,
time.time() - now)
return result
normalized_key = key.replace("/", "_")
normalized_key = normalized_key.replace(".", "_")
path = os.path.join(self.cache_dir, normalized_key)
if not self.force:
try:
raw_data = open(path).read()
if raw:
return raw_data
json_data = json.loads(raw_data)
result = json_serialization.load(json_data)
logging.debug("Cache hit %s", path)
return result
except (IOError, OSError):
pass
except Exception as e:
logging.error("Error loading from cache: %s" % e)
now = time.time()
result = cb(*args, **kwargs)
if desc:
logging.info("Completed %s in %d Seconds", desc, time.time() - now)
with open(path, "wb") as fd:
if raw:
fd.write(result)
else:
fd.write(rekall_utils.PPrint(json_serialization.dump(result)))
return result
def _make_profile(args):
"""A procedure that loads Pre-AST, computes a profile and stores it."""
config_text = open(args.config_file_path).read()
system_map_text = open(args.system_map_file_path).read()
logging.info('LOADING PREPROCESSOR AST FROM: %s', args.pre_ast_path)
preprocessed_ast = json_serialization.load_file(open(args.pre_ast_path))
logging.info('DONE')
if not preprocessed_ast:
raise RuntimeError("Unable to load pre-ast file.")
manager = layout_manager.ProfileBuilder(preprocessed_ast,
config_text,
system_map_text,
cache_dir=args.cache_dir)
profile = manager.create_profile(args.layouts_to_compute)
with open(args.output, "wb") as fd:
fd.write(rekall_utils.PPrint(profile))
def render(self, renderer):
spec = self.io_manager.GetData(self.spec)
renderer.write(utils.PPrint(self.build_index(spec)))
