def create_rekall_profiles(injector: Injector): tmp = None for file in dll_file_list: try: logging.info(f"Fetching rekall profile for {file.path}") local_dll_path = os.path.join(PROFILE_DIR, file.dest) guest_dll_path = str(PureWindowsPath("C:/", file.path)) cmd = injector.read_file(guest_dll_path, local_dll_path) out = json.loads(cmd.stdout.decode()) if out["Status"] == "Error" and out["Error"] in [ "ERROR_FILE_NOT_FOUND", "ERROR_PATH_NOT_FOUND" ]: raise FileNotFoundError if out["Status"] != "Success": logging.debug("stderr: " + cmd.stderr.decode()) logging.debug(out) # Take care if the error message is changed raise Exception("Some error occurred in injector") guid = pdb_guid(local_dll_path) tmp = fetch_pdb(guid["filename"], guid["GUID"], PROFILE_DIR) logging.debug("Parsing PDB into JSON profile...") profile = make_pdb_profile(tmp) with open(os.path.join(PROFILE_DIR, f"{file.dest}.json"), 'w') as f: f.write(profile) except json.JSONDecodeError: logging.debug(f"stdout: {cmd.stdout}") logging.debug(f"stderr: {cmd.stderr}") logging.debug(traceback.format_exc()) raise Exception(f"Failed to parse json response on {file.path}") except FileNotFoundError: logging.warning(f"Failed to copy file {file.path}, skipping...") except RuntimeError: logging.warning( f"Failed to fetch profile for {file.path}, skipping...") except Exception as e: # Take care if the error message is changed if str(e) == "Some error occurred in injector": raise else: logging.warning( f"Unexpected exception while creating rekall profile for {file.path}, skipping..." ) # Can help in debugging logging.debug("stderr: " + cmd.stderr.decode()) logging.debug(out) logging.debug(traceback.format_exc()) finally: safe_delete(local_dll_path) # was crashing here if the first file reached some exception if tmp is not None: safe_delete(os.path.join(PROFILE_DIR, tmp))
def create_rekall_profiles(install_info: InstallInfo): storage_backend = get_storage_backend(install_info) with storage_backend.vm0_root_as_block() as block_device, \ tempfile.TemporaryDirectory() as mount_path: mnt_path_quoted = shlex.quote(mount_path) blk_quoted = shlex.quote(block_device) try: subprocess.check_output( f"mount -t ntfs -o ro {blk_quoted} {mnt_path_quoted}", shell=True) except subprocess.CalledProcessError: raise RuntimeError(f"Failed to mount {block_device} as NTFS.") profiles_path = os.path.join(LIB_DIR, "profiles") for file in dll_file_list: try: logging.info(f"Fetching rekall profile for {file.path}") local_dll_path = os.path.join(profiles_path, file.dest) copyfile(os.path.join(mount_path, file.path), local_dll_path) guid = pdb_guid(local_dll_path) tmp = fetch_pdb(guid["filename"], guid["GUID"], profiles_path) logging.debug("Parsing PDB into JSON profile...") profile = make_pdb_profile(tmp) with open(os.path.join(profiles_path, f"{file.dest}.json"), 'w') as f: f.write(profile) except FileNotFoundError: logging.warning( f"Failed to copy file {file.path}, skipping...") except RuntimeError: logging.warning( f"Failed to fetch profile for {file.path}, skipping...") except Exception: logging.warning( f"Unexpected exception while creating rekall profile for {file.path}, skipping..." ) finally: if os.path.exists(local_dll_path): os.remove(local_dll_path) if os.path.exists(os.path.join(profiles_path, tmp)): os.remove(os.path.join(profiles_path, tmp)) # cleanup subprocess.check_output(f'umount {mnt_path_quoted}', shell=True)
def postinstall(report, generate_usermode): if os.path.exists(os.path.join(ETC_DIR, "no_usage_reports")): report = False install_info = InstallInfo.load() max_vms = install_info.max_vms output = subprocess.check_output(['vmi-win-guid', 'name', 'vm-0'], timeout=30).decode('utf-8') try: version = re.search(r'Version: (.*)', output).group(1) pdb = re.search(r'PDB GUID: ([0-9a-f]+)', output).group(1) fn = re.search(r'Kernel filename: ([a-z]+\.[a-z]+)', output).group(1) except AttributeError: logging.error("Failed to obtain kernel PDB GUID/Kernel filename.") return logging.info("Determined PDB GUID: {}".format(pdb)) logging.info("Determined kernel filename: {}".format(fn)) logging.info("Fetching PDB file...") dest = fetch_pdb(fn, pdb, destdir=os.path.join(LIB_DIR, 'profiles/')) logging.info("Generating profile out of PDB file...") profile = make_pdb_profile(dest) logging.info("Saving profile...") kernel_profile = os.path.join(LIB_DIR, 'profiles', 'kernel.json') with open(kernel_profile, 'w') as f: f.write(profile) output = subprocess.check_output( ['vmi-win-offsets', '--name', 'vm-0', '--json-kernel', kernel_profile], timeout=30).decode('utf-8') offsets = re.findall(r'^([a-z_]+):(0x[0-9a-f]+)$', output, re.MULTILINE) if not offsets: logging.error("Failed to parse output of vmi-win-offsets.") return offsets_dict = {k: v for k, v in offsets} if 'kpgd' not in offsets_dict: logging.error("Failed to obtain KPGD value.") return module_dir = os.path.dirname(os.path.realpath(__file__)) pid_tool = os.path.join(module_dir, "tools", "get-explorer-pid") explorer_pid_s = subprocess.check_output( [pid_tool, "vm-0", kernel_profile, offsets_dict['kpgd']], timeout=30).decode('ascii', 'ignore') m = re.search(r'explorer\.exe:([0-9]+)', explorer_pid_s) explorer_pid = m.group(1) runtime_profile = {"vmi_offsets": offsets_dict, "inject_pid": explorer_pid} logging.info("Saving runtime profile...") with open(os.path.join(LIB_DIR, 'profiles', 'runtime.json'), 'w') as f: f.write(json.dumps(runtime_profile, indent=4)) logging.info("Saving VM snapshot...") subprocess.check_output('xl save vm-0 ' + os.path.join(LIB_DIR, "volumes", "snapshot.sav"), shell=True) storage_backend = get_storage_backend(install_info) storage_backend.snapshot_vm0_volume() logging.info("Snapshot was saved succesfully.") if generate_usermode: try: create_rekall_profiles(install_info) except RuntimeError as e: logging.warning("Generating usermode profiles failed") logging.exception(e) for vm_id in range(max_vms + 1): # we treat vm_id=0 as special internal one generate_vm_conf(install_info, vm_id) if report: send_usage_report({ "kernel": { "guid": pdb, "filename": fn, "version": version }, "install_iso": { "sha256": install_info.iso_sha256 } }) reenable_services() logging.info("All right, drakrun setup is done.")
def generate_profiles(no_report=False): if os.path.exists(os.path.join(ETC_DIR, "no_usage_reports")): no_report = True with open(os.path.join(ETC_DIR, "install.json"), 'r') as f: install_info = json.loads(f.read()) max_vms = int(install_info["max_vms"]) output = subprocess.check_output(['vmi-win-guid', 'name', 'vm-0'], timeout=30).decode('utf-8') try: version = re.search(r'Version: (.*)', output).group(1) pdb = re.search(r'PDB GUID: ([0-9a-f]+)', output).group(1) fn = re.search(r'Kernel filename: ([a-z]+\.[a-z]+)', output).group(1) except AttributeError: logging.error("Failed to obtain kernel PDB GUID/Kernel filename.") return logging.info("Determined PDB GUID: {}".format(pdb)) logging.info("Determined kernel filename: {}".format(fn)) logging.info("Fetching PDB file...") dest = fetch_pdb(fn, pdb, destdir=os.path.join(LIB_DIR, 'profiles/')) logging.info("Generating profile out of PDB file...") profile = make_pdb_profile(dest) logging.info("Saving profile...") kernel_profile = os.path.join(LIB_DIR, 'profiles/kernel.json') with open(kernel_profile, 'w') as f: f.write(profile) output = subprocess.check_output( ['vmi-win-offsets', '--name', 'vm-0', '--json-kernel', kernel_profile], timeout=30).decode('utf-8') offsets = re.findall(r'^([a-z_]+):(0x[0-9a-f]+)$', output, re.MULTILINE) if not offsets: logging.error("Failed to parse output of vmi-win-offsets.") return offsets_dict = {k: v for k, v in offsets} if 'kpgd' not in offsets_dict: logging.error("Failed to obtain KPGD value.") return pid_tool = os.path.join(MAIN_DIR, "tools/get-explorer-pid") explorer_pid_s = subprocess.check_output( [pid_tool, "vm-0", kernel_profile, offsets_dict['kpgd']], timeout=30).decode('ascii', 'ignore') m = re.search(r'explorer\.exe:([0-9]+)', explorer_pid_s) explorer_pid = m.group(1) runtime_profile = {"vmi_offsets": offsets_dict, "inject_pid": explorer_pid} logging.info("Saving runtime profile...") with open(os.path.join(LIB_DIR, 'profiles/runtime.json'), 'w') as f: f.write(json.dumps(runtime_profile, indent=4)) # TODO (optional) making usermode profiles (a special tool for GUID extraction is required) logging.info("Saving VM snapshot...") subprocess.check_output('xl save vm-0 ' + os.path.join(LIB_DIR, "volumes/snapshot.sav"), shell=True) logging.info("Snapshot was saved succesfully.") if install_info["storage_backend"] == 'zfs': snap_name = shlex.quote( os.path.join(install_info["zfs_tank_name"], 'vm-0@booted')) subprocess.check_output(f'zfs snapshot {snap_name}', shell=True) for vm_id in range(max_vms + 1): # we treat vm_id=0 as special internal one generate_vm_conf(install_info, vm_id) if not no_report: send_usage_report({ "kernel": { "guid": pdb, "filename": fn, "version": version }, "install_iso": { "sha256": install_info["iso_sha256"] } }) reenable_services() logging.info("All right, drakrun setup is done.")
def create_rekall_profiles(install_info: InstallInfo): profiles_path = os.path.join(LIB_DIR, "profiles") with tempfile.TemporaryDirectory() as mount_path: # we mount 2nd partition, as 1st partition is windows boot related and 2nd partition is C:\\ if install_info.storage_backend == "zfs": # workaround for not being able to mount a snapshot base_snap = shlex.quote(os.path.join(install_info.zfs_tank_name, 'vm-0@booted')) tmp_snap = shlex.quote(os.path.join(install_info.zfs_tank_name, 'tmp')) try: subprocess.check_output(f'zfs clone {base_snap} {tmp_snap}', shell=True) except subprocess.CalledProcessError: logging.warning("Failed to clone temporary zfs snapshot. Aborting generation of usermode rekall profiles") return volume_path = os.path.join("/", "dev", "zvol", install_info.zfs_tank_name, "tmp-part2") # Wait for 60s for the volume to appear in /dev/zvol/... for _ in range(60): if os.path.exists(volume_path): break time.sleep(1.0) else: raise RuntimeError(f"ZFS volume not available at {volume_path}") try: # We have to wait for a moment for zvol to appear time.sleep(1.0) tmp_mount = shlex.quote(volume_path) subprocess.check_output(f'mount -t ntfs -o ro {tmp_mount} {mount_path}', shell=True) except subprocess.CalledProcessError: logging.warning("Failed to mount temporary zfs snapshot. Aborting generation of usermode rekall profiles") try: subprocess.check_output(f'zfs destroy {tmp_snap}', shell=True) except subprocess.CalledProcessError: logging.exception('Failed to cleanup after zfs tmp snapshot') return else: # qcow2 try: subprocess.check_output("modprobe nbd", shell=True) except subprocess.CalledProcessError: logging.warning("Failed to load nbd kernel module. Aborting generation of usermode rekall profiles") return # TODO: this assumes /dev/nbd0 is free try: subprocess.check_output(f"qemu-nbd -c /dev/nbd0 --read-only {os.path.join(LIB_DIR, 'volumes', 'vm-0.img')}", shell=True) except subprocess.CalledProcessError: logging.warning("Failed to load quemu image as nbd0. Aborting generation of usermode rekall profiles") return try: subprocess.check_output(f"mount -t ntfs -o ro /dev/nbd0p2 {mount_path}", shell=True) except subprocess.CalledProcessError: logging.warning("Failed to mount nbd0p2. Aborting generation of usermode rekall profiles") try: subprocess.check_output('qemu-nbd --disconnect /dev/nbd0', shell=True) except subprocess.CalledProcessError: logging.exception('Failed to cleanup after nbd0') return for file in dll_file_list: try: logging.info(f"Fetching rekall profile for {file.path}") local_dll_path = os.path.join(profiles_path, file.dest) copyfile(os.path.join(mount_path, file.path), local_dll_path) guid = pdb_guid(local_dll_path) tmp = fetch_pdb(guid["filename"], guid["GUID"], profiles_path) logging.debug("Parsing PDB into JSON profile...") profile = make_pdb_profile(tmp) with open(os.path.join(profiles_path, f"{file.dest}.json"), 'w') as f: f.write(profile) except FileNotFoundError: logging.warning(f"Failed to copy file {file.path}, skipping...") except RuntimeError: logging.warning(f"Failed to fetch profile for {file.path}, skipping...") except Exception: logging.warning(f"Unexpected exception while creating rekall profile for {file.path}, skipping...") finally: if os.path.exists(local_dll_path): os.remove(local_dll_path) if os.path.exists(os.path.join(profiles_path, tmp)): os.remove(os.path.join(profiles_path, tmp)) # cleanup subprocess.check_output(f'umount {mount_path}', shell=True) if install_info.storage_backend == "zfs": subprocess.check_output(f'zfs destroy {tmp_snap}', shell=True) else: # qcow2 subprocess.check_output('qemu-nbd --disconnect /dev/nbd0', shell=True)
def postinstall(report, generate_usermode): if not check_root(): return if os.path.exists(os.path.join(ETC_DIR, "no_usage_reports")): report = False install_info = InstallInfo.load() logging.info("Cleaning up leftovers(if any)") cleanup_postinstall_files() logging.info("Ejecting installation CDs") eject_cd("vm-0", FIRST_CDROM_DRIVE) if install_info.enable_unattended: # If unattended install is enabled, we have an additional CD-ROM drive eject_cd("vm-0", SECOND_CDROM_DRIVE) output = subprocess.check_output(['vmi-win-guid', 'name', 'vm-0'], timeout=30).decode('utf-8') try: version = re.search(r'Version: (.*)', output).group(1) pdb = re.search(r'PDB GUID: ([0-9a-f]+)', output).group(1) fn = re.search(r'Kernel filename: ([a-z]+\.[a-z]+)', output).group(1) except AttributeError: logging.error("Failed to obtain kernel PDB GUID/Kernel filename.") return logging.info("Determined PDB GUID: {}".format(pdb)) logging.info("Determined kernel filename: {}".format(fn)) logging.info("Fetching PDB file...") dest = fetch_pdb(fn, pdb, destdir=PROFILE_DIR) logging.info("Generating profile out of PDB file...") profile = make_pdb_profile(dest) logging.info("Saving profile...") kernel_profile = os.path.join(PROFILE_DIR, 'kernel.json') with open(kernel_profile, 'w') as f: f.write(profile) vmi_offsets = extract_vmi_offsets('vm-0', kernel_profile) explorer_pid = extract_explorer_pid('vm-0', kernel_profile, vmi_offsets) runtime_info = RuntimeInfo(vmi_offsets=vmi_offsets, inject_pid=explorer_pid) logging.info("Saving runtime profile...") with open(os.path.join(PROFILE_DIR, 'runtime.json'), 'w') as f: f.write(runtime_info.to_json(indent=4)) logging.info("Saving VM snapshot...") subprocess.check_output('xl save vm-0 ' + os.path.join(VOLUME_DIR, "snapshot.sav"), shell=True) storage_backend = get_storage_backend(install_info) storage_backend.snapshot_vm0_volume() logging.info("Snapshot was saved succesfully.") if generate_usermode: try: create_rekall_profiles(install_info) except RuntimeError as e: logging.warning("Generating usermode profiles failed") logging.exception(e) if report: send_usage_report({ "kernel": { "guid": pdb, "filename": fn, "version": version }, "install_iso": { "sha256": install_info.iso_sha256 } }) logging.info("All right, drakrun setup is done.") logging.info("First instance of drakrun will be enabled automatically...") subprocess.check_output('systemctl enable drakrun@1', shell=True) subprocess.check_output('systemctl start drakrun@1', shell=True) logging.info("If you want to have more parallel instances, execute:") logging.info(" # draksetup scale <number of instances>")
def postinstall(report, generate_usermode): if not check_root(): return if os.path.exists(os.path.join(ETC_DIR, "no_usage_reports")): report = False install_info = InstallInfo.load() storage_backend = get_storage_backend(install_info) vm0 = VirtualMachine(storage_backend, 0) if vm0.is_running is False: logging.exception("vm-0 is not running") return logging.info("Cleaning up leftovers(if any)") cleanup_postinstall_files() logging.info("Ejecting installation CDs") eject_cd("vm-0", FIRST_CDROM_DRIVE) if install_info.enable_unattended: # If unattended install is enabled, we have an additional CD-ROM drive eject_cd("vm-0", SECOND_CDROM_DRIVE) kernel_info = vmi_win_guid("vm-0") logging.info(f"Determined PDB GUID: {kernel_info.guid}") logging.info(f"Determined kernel filename: {kernel_info.filename}") logging.info("Fetching PDB file...") dest = fetch_pdb(kernel_info.filename, kernel_info.guid, destdir=PROFILE_DIR) logging.info("Generating profile out of PDB file...") profile = make_pdb_profile(dest) logging.info("Saving profile...") kernel_profile = os.path.join(PROFILE_DIR, "kernel.json") with open(kernel_profile, "w") as f: f.write(profile) safe_delete(dest) vmi_offsets = extract_vmi_offsets("vm-0", kernel_profile) explorer_pid = extract_explorer_pid("vm-0", kernel_profile, vmi_offsets) runtime_info = RuntimeInfo(vmi_offsets=vmi_offsets, inject_pid=explorer_pid) logging.info("Saving runtime profile...") with open(os.path.join(PROFILE_DIR, "runtime.json"), "w") as f: f.write(runtime_info.to_json(indent=4)) logging.info("Saving VM snapshot...") # Create vm-0 snapshot, and destroy it # WARNING: qcow2 snapshot method is a noop. fresh images are created on the fly # so we can't keep the vm-0 running vm0.save(os.path.join(VOLUME_DIR, "snapshot.sav")) logging.info("Snapshot was saved succesfully.") # Memory state is frozen, we can't do any writes to persistent storage logging.info("Snapshotting persistent memory...") storage_backend.snapshot_vm0_volume() if report: send_usage_report({ "kernel": { "guid": kernel_info.guid, "filename": kernel_info.filename, "version": kernel_info.version, }, "install_iso": { "sha256": install_info.iso_sha256 }, }) if generate_usermode: # Restore a VM and create usermode profiles create_missing_profiles() logging.info("All right, drakrun setup is done.") logging.info("First instance of drakrun will be enabled automatically...") subprocess.check_output("systemctl enable drakrun@1", shell=True) subprocess.check_output("systemctl start drakrun@1", shell=True) logging.info("If you want to have more parallel instances, execute:") logging.info(" # draksetup scale <number of instances>")
def create_rekall_profile(injector: Injector, file: DLL, raise_on_error=False): pdb_tmp_filepath = None cmd = None out = None try: logging.info(f"Fetching rekall profile for {file.path}") local_dll_path = os.path.join(PROFILE_DIR, file.dest) guest_dll_path = str(PureWindowsPath("C:/", file.path)) cmd = injector.read_file(guest_dll_path, local_dll_path) out = json.loads(cmd.stdout.decode()) if out["Status"] == "Error" and out["Error"] in [ "ERROR_FILE_NOT_FOUND", "ERROR_PATH_NOT_FOUND", ]: raise FileNotFoundError if out["Status"] != "Success": logging.debug("stderr: " + cmd.stderr.decode()) logging.debug(out) # Take care if the error message is changed raise Exception("Some error occurred in injector") static_apiscout_dll_profile = make_static_apiscout_profile_for_dll( local_dll_path) with open(os.path.join(APISCOUT_PROFILE_DIR, f"{file.dest}.json"), "w") as f: f.write( json.dumps(static_apiscout_dll_profile, indent=4, sort_keys=True)) codeview_data = pe_codeview_data(local_dll_path) pdb_tmp_filepath = fetch_pdb(codeview_data["filename"], codeview_data["symstore_hash"], PROFILE_DIR) logging.debug("Parsing PDB into JSON profile...") profile = make_pdb_profile( pdb_tmp_filepath, dll_origin_path=guest_dll_path, dll_path=local_dll_path, dll_symstore_hash=codeview_data["symstore_hash"], ) with open(os.path.join(PROFILE_DIR, f"{file.dest}.json"), "w") as f: f.write(profile) except json.JSONDecodeError: logging.debug(f"stdout: {cmd.stdout}") logging.debug(f"stderr: {cmd.stderr}") logging.debug(traceback.format_exc()) raise Exception(f"Failed to parse json response on {file.path}") except FileNotFoundError as e: on_create_rekall_profile_failure(f"Failed to copy file {file.path}", raise_on_error, e) except RuntimeError as e: on_create_rekall_profile_failure( f"Failed to fetch profile for {file.path}", raise_on_error, e) except subprocess.TimeoutExpired as e: on_create_rekall_profile_failure(f"Injector timed out for {file.path}", raise_on_error, e) except Exception as e: # Take care if the error message is changed if str(e) == "Some error occurred in injector": raise else: # Can help in debugging if cmd: logging.debug("stdout: " + cmd.stdout.decode()) logging.debug("stderr: " + cmd.stderr.decode()) logging.debug("rc: " + str(cmd.returncode)) logging.debug(traceback.format_exc()) on_create_rekall_profile_failure( f"Unexpected exception while creating rekall profile for {file.path}", raise_on_error, e, ) finally: safe_delete(local_dll_path) # was crashing here if the first file reached some exception if pdb_tmp_filepath is not None: safe_delete(os.path.join(PROFILE_DIR, pdb_tmp_filepath))