def fog_init(self):
"""Initiate by indexing FOG regarding all available machines."""
self.fog_machines = {}
if self.options.fog.hostname == "none":
return
# TODO Handle exceptions such as not being able to connect.
# Parse the HTML.
r = requests.get("http://" + self.options.fog.hostname + "/fog/status", headers=headers, verify=False)
if r.status_code != 200:
raise CuckooCriticalError("The FOG server answered with the status code " + str(r.status_code))
r_hosts = requests.get("http://" + self.options.fog.hostname + "/fog/host", headers=headers, verify=False)
hosts = r_hosts.json()['hosts']
hostnames = []
for host in hosts:
hostnames.append(host['name'])
print("Host " + host['name'] + " has MAC " + host['macs'][0])
# Check whether all our machines are available on FOG.
for machine in self.machines():
if machine.label not in hostnames:
raise CuckooMachineError(
"The physical machine %s has not been defined in FOG, "
"please investigate and configure the configuration "
"correctly." % machine.label
)
def stop(self, label):
"""Stops a physical machine.
@param label: physical machine name.
@raise CuckooMachineError: if unable to stop.
"""
# Since we are 'stopping' a physical machine, it must
# actually be rebooted to kick off the reimaging process
n = self.options.physical.user
p = self.options.physical.password
creds = str(n) + '%' + str(p)
status = self._status(label)
if status == self.RUNNING:
log.debug("Rebooting machine: %s." % label)
machine = self._get_machine(label)
shutdown = subprocess.Popen([
'net', 'rpc', 'shutdown', '-I', machine.ip, '-U', creds, '-r',
'-f', '--timeout=5'
],
stdout=subprocess.PIPE)
output = shutdown.communicate()[0]
if not "Shutdown of remote machine succeeded" in output:
raise CuckooMachineError('Unable to initiate RPC request')
else:
log.debug("Reboot success: %s." % label)
def stop(self, label):
"""Stops a physical machine.
@param label: physical machine name.
@raise CuckooMachineError: if unable to stop.
"""
# Since we are 'stopping' a physical machine, it must
# actually be rebooted to kick off the re-imaging process.
creds = "%s%%%s" % (
self.options.physical.user, self.options.physical.password
)
if self._status(label) == self.RUNNING:
log.debug("Rebooting machine: %s.", label)
machine = self._get_machine(label)
args = [
"net", "rpc", "shutdown", "-I", machine.ip,
"-U", creds, "-r", "-f", "--timeout=5"
]
output = subprocess.check_output(args)
if "Shutdown of remote machine succeeded" not in output:
raise CuckooMachineError("Unable to initiate RPC request")
else:
log.debug("Reboot success: %s." % label)
# Deploy a clean image through FOG, assuming we're using FOG.
self.fog_queue_task(label)
def fog_init(self):
"""Initiate by indexing FOG regarding all available machines."""
self.fog_machines = {}
if not HAVE_FOG or self.options.fog.hostname == "none":
return
# TODO Handle exceptions such as not being able to connect.
r = self.fog_query("node=tasks&sub=listhosts")
# Parse the HTML.
b = bs4.BeautifulSoup(r.content, "html.parser")
if not b.find_all("table"):
raise CuckooCriticalError(
"The supplied FOG username and/or password do not allow us "
"to login into FOG, please configure the correct credentials."
)
# Mapping for physical machine hostnames to their mac address and uri
# for "downloading" a safe image onto the host. Great piece of FOG API
# usage here.
for row in b.find_all("table")[0].find_all("tr")[1:]:
hostname, macaddr, download, upload, advanced = row.find_all("td")
self.fog_machines[hostname.text] = (
macaddr.text, next(download.children).attrs["href"][1:],
)
# Check whether all our machines are available on FOG.
for machine in self.machines():
if machine.label not in self.fog_machines:
raise CuckooMachineError(
"The physical machine %s has not been defined in FOG, "
"please investigate and configure the configuration "
"correctly." % machine.label
)
def _initialize_check(self):
"""Ensure that credentials have been entered into the config file.
@raise CuckooCriticalError: if no credentials were provided or if
one or more physical machines are offline.
"""
# TODO This should be moved to a per-machine thing.
if not self.options.physical.user or not self.options.physical.password:
raise CuckooCriticalError(
"Physical machine credentials are missing, please add it to "
"the Physical machinery configuration file."
)
global headers
headers = {
"fog-api-token": self.options.fog.apikey,
"fog-user-token": self.options.fog.user_apikey,
"Content-Type": "application/json"
}
self.fog_init()
for machine in self.machines():
status = self._status(machine.label)
if status == self.STOPPED:
# Send a Wake On Lan message (if we're using FOG).
self.wake_on_lan(machine.label)
elif status == self.ERROR:
raise CuckooMachineError(
"Unknown error occurred trying to obtain the status of "
"physical machine %s. Please turn it on and check the "
"Cuckoo Agent." % machine.label
)
def _wait_status(self, label, state):
"""Waits for a vm status.
@param label: virtual machine name.
@param state: virtual machine status, accepts multiple states as list.
@raise CuckooMachineError: if default waiting timeout expire.
"""
# This block was originally suggested by Loic Jaquemet.
waitme = 0
try:
current = self._status(label)
except NameError:
return
if isinstance(state, str):
state = [state]
while current not in state:
log.debug(
"Waiting %i cuckooseconds for machine %s to switch "
"to status %s", waitme, label, state)
if waitme > int(self.options_globals.timeouts.vm_state):
raise CuckooMachineError("Timeout hit while for machine {0} "
"to change status".format(label))
time.sleep(1)
waitme += 1
current = self._status(label)
def _list(self):
"""Lists virtual machines installed.
@return: virtual machine names list.
"""
try:
proc = subprocess.Popen(
[self.options.virtualbox.path, "list", "vms"],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
output = proc.communicate()
except OSError as e:
raise CuckooMachineError("VBoxManage error listing "
"installed machines: %s" % e)
machines = []
for line in output[0].split("\n"):
try:
label = line.split('"')[1]
if label == "<inaccessible>":
log.warning("Found an inaccessible vitual machine: "
"please check his state")
else:
machines.append(label)
except IndexError:
continue
return machines
def _list(self):
"""Lists virtual machines installed.
@return: virtual machine names list.
"""
try:
proc = subprocess.Popen(
[self.options.virtualbox.path, "list", "vms"],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
)
output, _ = proc.communicate()
except OSError as e:
raise CuckooMachineError(
f"VBoxManage error listing installed machines: {e}")
machines = []
for line in output.split("\n"):
try:
label = line.split('"', 2)[1]
if label == "<inaccessible>":
log.warning(
"Found an inaccessible virtual machine, please check its state"
)
else:
machines.append(label)
except IndexError:
continue
return machines
def stop(self, label):
"""Stops a physical machine.
@param label: physical machine name.
@raise CuckooMachineError: if unable to stop.
"""
# Since we are 'stopping' a physical machine, it must
# actually be rebooted to kick off the re-imaging process
n = self.options.physical.user
p = self.options.physical.password
creds = "%s%%%s" % (n, p)
status = self._status(label)
if status == self.RUNNING:
log.debug("Rebooting machine: %s.", label)
machine = self._get_machine(label)
args = [
"net", "rpc", "shutdown", "-I", machine.ip, "-U", creds, "-r",
"-f", "--timeout=5"
]
shutdown = subprocess.Popen(args, stdout=subprocess.PIPE)
output, _ = shutdown.communicate()
if "Shutdown of remote machine succeeded" not in output:
raise CuckooMachineError("Unable to initiate RPC request")
else:
log.debug("Reboot success: %s." % label)
def _check_disks_reset(self, vm):
"""Check whether each attached disk is set to reset on boot.
@param vm: vm record
"""
for ref in vm["VBDs"]:
try:
vbd = self.session.xenapi.VBD.get_record(ref)
except:
log.warning("Invalid VBD for vm %s: %s", vm["uuid"], ref)
continue
if vbd["type"] == "Disk":
vdi_ref = vbd["VDI"]
try:
vdi = self.session.xenapi.VDI.get_record(vdi_ref)
except:
log.warning("Invalid VDI for vm %s: %s", vm["uuid"],
vdi_ref)
continue
if vdi["on_boot"] != "reset" and vdi["read_only"] is False:
raise CuckooMachineError(
"Vm %s contains invalid VDI %s: disk is not reset on "
"boot. Please set the on-boot parameter to 'reset'." %
(vm["uuid"], vdi["uuid"]))
def find_snapshot(self, label, vm):
"""Find a specific or the most current snapshot of a VM.
@param label: VM label for additional parameter retrieval
@raise CuckooMachineError: if snapshots cannot be enumerated."""
# use a statically configured snapshot name if configured without any
# additional checks. User has to make sure it exists then.
snapshot = self.db.view_machine_by_label(label).snapshot
if snapshot:
return snapshot
# heuristically determine the most recent snapshot if no snapshot name
# is explicitly configured.
log.debug("%s: No snapshot configured, determining most recent one",
label)
try:
snapshots = vm.snapshot.get()
except ResourceException as e:
raise CuckooMachineError(f"Error enumerating snapshots: {e}")
snaptime = 0
snapshot = None
for snap in snapshots:
# ignore "meta-snapshot" current which is the current state
if snap["name"] == "current":
continue
if snap["snaptime"] > snaptime:
snaptime = snap["snaptime"]
snapshot = snap["name"]
return snapshot
def _wait_task(self, task):
"""Wait for a task to complete with timeout"""
limit = timedelta(seconds=int(cfg.timeouts.vm_state))
start = datetime.utcnow()
while True:
if task.info.state == "error":
raise CuckooMachineError("Task error")
if task.info.state == "success":
break
if datetime.utcnow() - start > limit:
raise CuckooMachineError("Task timed out")
time.sleep(1)
def stop(self, label):
"""Stops a virtual machine.
@param label: virtual machine label.
@raise CuckooMachineError: if unable to stop.
"""
log.debug("Stopping vm %s" % label)
vm_info = self.db.view_machine_by_label(label)
if self._status(vm_info.name) == self.STOPPED:
raise CuckooMachineError("Trying to stop an already stopped vm %s" % label)
proc = self.state.get(vm_info.name, None)
proc.kill()
stop_me = 0
while proc.poll() is None:
if stop_me < int(self.options_globals.timeouts.vm_state):
time.sleep(1)
stop_me += 1
else:
log.debug("Stopping vm %s timeouted. Killing" % label)
proc.terminate()
time.sleep(1)
# if proc.returncode != 0 and stop_me < int(self.options_globals.timeouts.vm_state):
# log.debug("QEMU exited with error powering off the machine")
self.state[vm_info.name] = None
def dump_memory(self, label, path):
"""Takes a memory dump.
@param path: path to where to store the memory dump.
"""
try:
proc = subprocess.Popen([self.options.virtualbox.path, "-v"],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
output, err = proc.communicate()
if proc.returncode != 0:
# It's quite common for virtualbox crap utility to exit with:
# VBoxManage: error: Details: code E_ACCESSDENIED (0x80070005)
# So we just log to debug this.
log.debug(
"VBoxManage returns error checking status for "
"machine %s: %s", label, err)
except OSError as e:
raise CuckooMachineError(
"VBoxManage failed return it's version: %s" % (e))
if output[:1] == str(5):
# VirtualBox version 5.x
dumpcmd = "dumpvmcore"
else:
# VirtualBox version 4.x
dumpcmd = "dumpguestcore"
try:
subprocess.call([
self.options.virtualbox.path, "debugvm", label, dumpcmd,
"--filename", path
],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True)
log.info(
"Successfully generated memory dump for virtual machine "
"with label %s to path %s", label, path)
except OSError as e:
raise CuckooMachineError("VBoxManage failed to take a memory "
"dump of the machine with label %s: %s" %
(label, e))
def dump_memory(self, vmx_path, path):
"""Take a memory dump of the machine."""
if not os.path.exists(vmx_path):
raise CuckooMachineError(
"Can't find .vmx file {0}. Ensure to configure a fully qualified path in vmware.conf (key = vmx_path)"
.format(vmx_path))
try:
subprocess.call(
[self.options.vmware.path, "snapshot", vmx_path, "memdump"],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
except OSError as e:
raise CuckooMachineError(
"vmrun failed to take a memory dump of the machine with label %s: %s"
% (vmx_path, e))
vmwarepath, _ = os.path.split(vmx_path)
latestvmem = max(glob.iglob(os.path.join(vmwarepath, "*.vmem")),
key=os.path.getctime)
# We need to move the snapshot to the current analysis directory as
# vmware doesn't support an option for the destination path :-/
shutil.move(latestvmem, path)
# Old snapshot can be deleted, as it isn't needed any longer.
try:
subprocess.call(
[
self.options.vmware.path, "deleteSnapshot", vmx_path,
"memdump"
],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
)
except OSError as e:
raise CuckooMachineError(
"vmrun failed to delete the temporary snapshot in %s: %s" %
(vmx_path, e))
log.info(
"Successfully generated memory dump for virtual machine with label %s ",
vmx_path)
def _is_running(self, vmx_path):
"""Checks if virtual machine is running.
@param vmx_path: path to vmx file
@return: running status
"""
try:
p = subprocess.Popen(
[self.options.vmware.path, "list"], universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE
)
output, error = p.communicate()
except OSError as e:
raise CuckooMachineError(f"Unable to check running status for {vmx_path}: {e}")
else:
if output:
return vmx_path in output
else:
raise CuckooMachineError(f"Unable to check running status for {vmx_path}, no output from `vmrun list`")
def _revert(self, vmx_path, snapshot):
"""Revets machine to snapshot.
@param vmx_path: path to vmx file
@param snapshot: snapshot name
@raise CuckooMachineError: if unable to revert
"""
log.debug("Revert snapshot for vm %s", vmx_path)
try:
if subprocess.call(
[self.options.vmware.path, "revertToSnapshot", vmx_path, snapshot],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
):
raise CuckooMachineError(f"Unable to revert snapshot for machine {vmx_path}: vmrun exited with error")
except OSError as e:
raise CuckooMachineError(f"Unable to revert snapshot for machine {vmx_path}: {e}")
def _connect(self):
"""Connects to libvirt subsystem.
@raise CuckooMachineError: if cannot connect to libvirt.
"""
try:
return libvirt.open("qemu:///system")
except libvirt.libvirtError:
raise CuckooMachineError("Cannot connect to libvirt")
def wait_for_task(self, taskid, label, vm, node):
"""Wait for long-running Proxmox task to finish.
@param taskid: id of Proxmox task to wait for
@raise CuckooMachineError: if task status cannot be determined."""
elapsed = 0
while elapsed < self.timeout:
try:
task = node.tasks(taskid).status.get()
except ResourceException as e:
raise CuckooMachineError(f"Error getting status of task {taskid}: {e}")
# extract operation name from task status for display
operation = task["type"]
if operation.startswith("qm"):
operation = operation[2:]
if task["status"] != "stopped":
log.debug("%s: Waiting for operation %s (%s) to finish", label, operation, taskid)
time.sleep(1)
elapsed += 1
continue
# VMs sometimes remain locked for some seconds after a task
# completed. They will get stuck in that state if another operation
# is attempted. So query the current VM status to extract the lock
# status.
try:
status = vm.status.current.get()
except ResourceException as e:
raise CuckooMachineError(f"Couldn't get status: {e}")
if "lock" in status:
log.debug("%s: Task finished but VM still locked", label)
if status["lock"] != operation:
log.warning("%s: Task finished but VM locked by different operation: %s", label, operation)
time.sleep(1)
elapsed += 1
continue
# task is really, really done
return task
# timeout expired
return None
def start(self, label):
"""Start a virtual machine.
@param label: virtual machine name.
@raise CuckooMachineError: if unable to start.
"""
log.debug("Starting vm %s", label)
if self._status(label) == self.RUNNING:
raise CuckooMachineError(f"Trying to start an already started vm {label}")
vm_info = self.db.view_machine_by_label(label)
virtualbox_args = [self.options.virtualbox.path, "snapshot", label]
if vm_info.snapshot:
log.debug("Using snapshot %s for virtual machine %s", vm_info.snapshot, label)
virtualbox_args.extend(["restore", vm_info.snapshot])
else:
log.debug("Using current snapshot for virtual machine %s", label)
virtualbox_args.extend(["restorecurrent"])
try:
if subprocess.call(
virtualbox_args, universal_newlines=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE, close_fds=True
):
raise CuckooMachineError("VBoxManage exited with error restoring the machine's snapshot")
except OSError as e:
raise CuckooMachineError(f"VBoxManage failed restoring the machine: {e}")
self._wait_status(label, self.SAVED)
try:
proc = subprocess.Popen(
[self.options.virtualbox.path, "startvm", label, "--type", self.options.virtualbox.mode],
universal_newlines=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
close_fds=True,
)
output, err = proc.communicate()
if err:
raise OSError(err)
except OSError as e:
raise CuckooMachineError(f"VBoxManage failed starting the machine in {self.options.virtualbox.mode.upper()} mode: {e}")
self._wait_status(label, self.RUNNING)
