def boot(mac):
# This function is called whenever a client performs a request to the /boot/<mac> entrypoint.
# when this happens, flask calls appropriate methods of the BareMetalManager
if not validate_mac(mac):
return "#!ipxe\nEcho INVALID MAC ADDRESS PROVIDED: %s" % mac, 400
else:
mac = MacAddress.MacAddress(mac) # type: MacAddress.MacAddress
# Check if the mac is among the registered machines. If so, revert it.
mgr = flask.current_app.config['bare_metal_manager']
machine = mgr.get_guest_by_mac(mac)
if machine is None:
logging.warn(
"Web boot: Machine with mac %s does not exist on this manager."
% mac)
return (
'#!ipxe\nEcho Mac address %s not registered with the manager.'
% mac, 404)
else:
logging.info("Wb boot: requested boot of machine with mac %s." %
mac)
# Given a mac, calculate target name and VHD paths
name, diff_path, base_path = mgr._calculate_iscsi_params(mac)
# Remove / recreate the iscsi target
iqn = create_differencing_disk(name, base_path, diff_path, mac)
# TODO: use the new templates
# Now compose the iPXE script file to be returned to the caller
data = IPXE_TEMPLATE.format(ip=mgr._iscsi_server_ip, iqn=iqn)
return data, 200
def create_guest(self):
"""
Creates a Virtual Machine with a differencing disk based on an immutable image.
The machine is not started.
:return: IMachine
"""
conn = self.get_connection()
name = str(uuid4())
logging.info("Creating VM %s" % name)
srv = conn.compute.create_server(name=name,
image_id=self._guest_image_id,
flavor_id=self._guest_flavor_id,
networks=[{
'uuid':
self._internal_network_id
}],
security_groups=[{
'name':
self._guest_security_group
}],
metadata={"IS_ANALYZER": "true"})
logging.debug("Waiting for vm %s ..." % name)
conn.compute.wait_for_server(srv, wait=180)
mac = MacAddress.MacAddress(srv.addresses[self._internal_network][0]
['OS-EXT-IPS-MAC:mac_addr'])
if self._status_handler is not None:
self._status_handler.register_worker(mac, WorkerStatus.IDLE)
logging.info("Machine %s created correctly." % name)
return OSMachine(manager=self, mac=mac, id=srv.id)
def get_netlog(
self,
machine, # type: OSMachine
directory # type:str
):
# This method may take FOREVER due to the great amount of data to be downloaded and analyzed.
# So we hold the lock just for the strict needed time.
mac = None
if not isinstance(machine, OSMachine):
raise Exception("Machine must be a reference to OSMachine.")
with machine.lock:
conn = self.get_connection()
m = None
try:
m = conn.compute.get_server(machine.get_id())
except:
raise Exception("Cannot find machine %s" + machine.get_id())
mac = MacAddress.MacAddress(
str(m.addresses[self._internal_network][0].get(
'OS-EXT-IPS-MAC:mac_addr')))
pcap_file = os.path.join(directory, NETLOG_NAME)
self._netmon.collect(mac, pcap_file)
cap_file_https = os.path.join(directory, HTTPS_NETLOG_NAME)
self._netmon.collect_https(mac, cap_file_https)
return pcap_file, cap_file_https
def list_guests(self):
"""
Returns the list of all Machines handled by this manager.
This implementation will return all the VM belonging to the group specified
by the constructor (vm_group)
:return: IMachine[]
"""
res = []
#with self.lock:
"""
d = dict()
self._getMgr(d)
mgr = d['mgr']
vbox = d['vbox']
"""
ctx = self.getMgr()
vms = ctx.vbox.getMachinesByGroups([self._vm_group])
for v in vms:
t = VBoxMachine(manager=self,
mac=MacAddress.MacAddress(
v.getNetworkAdapter(0).MACAddress),
id=v.id)
res.append(t)
return res # type: list[VBoxMachine]
def start(self, mac):
mac = MacAddress.MacAddress(mac)
# Check if there already is a sniffer working for this mac
response = requests.get(
urljoin(self._url, self.GET_SNIFFER.format(mac)))
# Is there already a sniffer for this traffic?
if response.status_code == 404:
# Need to prepare the sniffer
self._prepare(mac)
elif response.status_code == 200:
# Is the sniffer busy?
data = response.json()
if data['status'] == 'running':
# The sniffer is busy now. We can't proceed
raise SnifferAlreadyRunningException(
"Sniffer status isn't either prepared nor finished.")
else:
raise Exception(
"Unexpected return code %d. Check server log for error details."
% response.status_code)
# Let's start the sniffer
response = requests.get(
urljoin(self._url, self.START_SNIFFER.format(mac)))
if response.status_code != 200:
raise Exception(
"Unexpected result code %d. Check server log for error details."
% response.status_code)
def lookup_by_mac(self, mac):
# Lookup the given mac address from our dictionary
if mac is None:
return None
m = MacAddress.MacAddress(mac)
return self._handled_machines.get(m)
def stop(self, mac):
mac = MacAddress.MacAddress(mac)
response = requests.get(
urljoin(self._url, self.STOP_SNIFFER.format(mac)))
if response.status_code != 200:
raise Exception(
"Unexpected return code %d. Check server log for error details."
% response.status_code)
def __init__(self, manager, mac, smartplug_ip):
if not isinstance(manager, BareMetalManager):
raise Exception(
"BareMetalMachine can be registered only to a BareMetalManager."
)
self._manager = manager
self._mac = MacAddress.MacAddress(mac)
self._smartplug_ip = str(smartplug_ip)
self._smart_plug = HS1XX(ip=self._smartplug_ip)
def __init__(self,
diff_vhd_folder,
base_vhd_path,
sniffer_url,
iscsi_server_ip,
machines_conf,
external_hc_ip,
external_hc_port,
binding_host,
binding_port=8181):
super(BareMetalManager, self).__init__()
self._child_vhd_folder = diff_vhd_folder
self._base_vhd_path = base_vhd_path
self._netmon = BareNetMon(sniffer_url)
self._iscsi_server_ip = iscsi_server_ip
self._binding_host = binding_host
self._binding_port = binding_port
self._ipxe_server = FlaskWrapper(host=self._binding_host,
port=self._binding_port,
manager=self)
self._external_hc_ip = external_hc_ip
self._external_hc_port = external_hc_port
machines = None
# If the machines parameter is provided in string form, load it into a convenient json obj
if isinstance(machines_conf, str):
try:
machines = json.loads(machines_conf)
except:
raise ValueError(
"Invalid machines_conf parameter specified. Such parameter must either be a valid json string or a dictionary"
)
# Now validate its schema
try:
validate(machines, schema)
except Exception:
raise ValueError(
"Invalid machines_conf parameter specified. "
"Json object was not in a correct form. Please specify it as an array of objects, each one"
" containing two properties: sandbox_mac (mac address as string) and smartplug_ip "
"(ip address as string)")
# Ok, load the machines
for m in machines:
tmp = BareMetalMachine(manager=self,
mac=MacAddress.MacAddress(m['sandbox_mac']),
smartplug_ip=str(m['smartplug_ip']))
self._machines.append(tmp)
def add_manager(
self,
m_manager # type:IGuestManager
):
"""
Adds a machine amanager (implementing IMachineManager interface) to the list of managers. By doing so, the
the main manager can perform hybrid analysis, taking advantage of multiple-distinct hypervisor technologies.
:param mgr:
:return:
"""
self._managers.append(m_manager)
for m in m_manager.list_guests():
self._handled_machines[MacAddress.MacAddress(m._mac)] = m
def validate_mac(mac):
"""
Simply checks if a given input string is a valid mac address
:param mac:
:return:
"""
if mac is None:
return False
try:
m = MacAddress.MacAddress(mac)
return True
except:
return False
def analyse(self, mac, dest_file):
mac = MacAddress.MacAddress(mac)
response = requests.get(
urljoin(self._url, self.COLLECT_ANALYSIS.format(mac)))
if response.status_code != 200:
raise Exception(
"Unexpected return code %d. Check server log for error details."
% response.status_code)
# The expected data is in binary format, representing the log catch by the sniffer in pcap format.
with open(dest_file, 'wb') as f:
f.write(response.content)
return dest_file
def __init__(
self,
manager, # type:IGuestManager
mac, # type:MacAddress
id # type:str
):
if not isinstance(manager, VBoxManager):
raise Exception(
"VBoxMachine requires VBoxManger to be specified as manager")
self._manager = manager
self._mac = MacAddress.MacAddress(mac)
self._id = str(id)
self.lock = LogLock("VboxMachine %s" % self._mac)
def start_network_sniffing(
self,
guest # type: BareMetalMachine
):
mac = MacAddress.MacAddress(guest.get_mac())
logging.debug("Staring sniffer/network monitor for mac %s" % mac)
try:
self._netmon.start(mac=mac)
except SnifferAlreadyRunningException as ex:
logging.warn(
"Sniffer for mac %s was already running. I will stop it and start it again."
% mac)
self._netmon.stop(mac=mac)
self._netmon.start(mac=mac)
def _prepare(self, mac):
mac = MacAddress.MacAddress(mac)
data = {
"mac": str(mac)
# The following are not necessary since the HostController will register itself with the sniffer at boot.
#,"hc_ip": get_local_ip_routing_to_addr(CFG.network_analyzer_ip)
#,"hc_port": CFG.bind_host_port
}
response = requests.post(urljoin(self._url, self.POST_SNIFFER),
data=json.dumps(data))
if response.status_code != 201:
raise Exception(
"Unexpected return code %d. Check server log for error details."
% response.status_code)
def get_guest_by_mac(
self,
mac # type: MacAddress
):
"""
If the given mac matches any VM handled by this manager, an instance
of the VM is returned. Otherwise None is returned.
:return:
"""
mac = MacAddress.MacAddress(mac)
vms = self.list_guests()
for v in vms:
if v.get_mac() == mac:
return v # type: BareMetalMachine
return None
def query_status(self, mac):
"""
Returns the status of the current sniffer. If no sniffer is found None is returned.
:param mac:
:return:
"""
mac = MacAddress.MacAddress(mac)
response = requests.get(
urljoin(self._url, self.GET_SNIFFER.format(mac)))
if response.status_code == 404:
return None
if response.status_code != 200:
raise Exception(
"Unexpected return code %d. Check server log for error details."
% response.status_code)
return response.json()['status']
def run(self):
jmgr = db.jobmanager
# Deamon!
while self.ctrl.should_run():
# Select all the workers from the DB that are tacking more than expected
# Kill every VM related
# Report Failure
# Remove worker records
# Wait interval
dead = jmgr.get_pending_workers(self.timeout)
if len(dead) > 0:
wd_log.warning(
"Watchdog found %d workers above the timeout. I am going to terminate those "
"VM and report failure for associated experiment." %
len(dead))
for w in dead:
mac = MacAddress.MacAddress(w.mac)
try:
# Report failure for this job. We try to address any possible strange situation in here.
# This method should be as robust as possible
exp_id = db.lookup_experiment_id_by_work_id(w.id)
if exp_id is None:
wd_log.warning(
"Found a pending worker with null associated experiment_id. "
"Such worker will be terminated.")
# Stop associated machine
machine = self.ctrl.lookup_by_mac(mac)
mm = machine.get_manager()
mm.stop_guest(machine)
wd_log.info("Watchdog stopped machine with mac %s." % mac)
# Now update the db accordingly
jmgr.set_work_error(
w.id, info="Terminated by watchdog to to timeout.")
# Finally revert the machine
mm.revert_guest(machine)
except Exception:
wd_log.exception("Watchdog got an exception.")
# We do not crash now.
# Run again later...
time.sleep(self.run_interval)
def _calculate_iscsi_params(self, mac):
"""
Given a mac address string as input, calculates the iscsi target name and the path of the differencing vhd to be created
:param mac:
:return:
"""
if mac is None:
raise ValueError("Invalid mac address provided")
name = None
name = str(MacAddress.MacAddress(mac))
name = name.lower().strip()
name = name.replace("-", "")
name = name.replace(":", "")
return name, os.path.join(self._child_vhd_folder,
name + ".vhdx"), self._base_vhd_path
def stop_network_sniffing(
self,
guest # type: BareMetalMachine
):
# Stop the sniffer, if present
mac = MacAddress.MacAddress(guest.get_mac())
logging.debug(
"Stopping associated network sniffers to (traffic for/from %s)" %
mac)
status = self._netmon.query_status(mac)
if status is None:
logging.debug("There is no sniffer for mac %s" % mac)
else:
logging.debug("Sniffer status is %s" % status)
if status == "running":
logging.debug("Stopping sniffer for mac %s" % mac)
self._netmon.stop(mac)
def get_netlog(
self,
machine, # type: VBoxMachine
directory # type:str
):
# This method may take FOREVER due to the great amount of data to be downloaded and analyzed.
# So we hold the lock just for the strict needed time.
mac = None
if not isinstance(machine, VBoxMachine):
raise Exception("Machine must be a reference to VBoxMachine.")
with machine.lock:
"""
d = dict()
self._getMgr(d)
mgr = d['mgr']
vbox = d['vbox']
"""
ctx = self.getMgr()
id = machine.get_id()
m = None
try:
m = ctx.vbox.findMachine(machine.get_id())
except:
raise Exception("Cannot find machine " + id)
mac = MacAddress.MacAddress(m.getNetworkAdapter(0).MACAddress)
pcap_file = os.path.join(directory, NETLOG_NAME)
self._netmon.collect(mac, pcap_file)
cap_file_https = os.path.join(directory, HTTPS_NETLOG_NAME)
self._netmon.collect_https(mac, cap_file_https)
return pcap_file, cap_file_https
def notify_machine_status(self, machine_mac, worker_status):
mac = MacAddress.MacAddress(machine_mac)
with self._lock:
db.jobmanager.update_worker_status(worker_mac=mac,
hc_id=CFG.host_controller_id,
status=worker_status)
def create_differencing_disk(iscsi_target_name,
parent_vhdx_path,
child_vhdx_path,
initiator_mac,
iqn=None):
import pythoncom
pythoncom.CoInitialize()
initiator_mac = MacAddress.MacAddress(initiator_mac)
connection = wmi.WMI(moniker='//./root/wmi')
conn_cimv2 = wmi.WMI(moniker='//./root/cimv2')
# Make sure we are on Win platform and we can connect to the WMI service
_ensure_wt_provider_available(connection)
"""
Creates a new diff_disk_vhdx extending the given parent_vhdx_path, and registers it to the specified isci target
with the given iqn. If child disk exists, it will be disconnected, removed and created again. Returns the IQN of the
newly created target.
:param iscsi_target_name:
:param iqn:
:param parent_vhdx_path:
:param child_vhdx_path:
:return:
"""
# NOTE! The MS iSCSI service only supports VHDX format. If you try to use any other format you'll get a stupid
# non-sense error. So we make sure the parent path points to a VHDX file.
_check_parent_image(parent_vhdx_path)
# Retrieve any WT_DISK associated with the child image disk, if any.
wtd_child = _get_wt_disk(connection, child_vhdx_path)
# In case we found anyone, make sure there is no lun attached to it. If so, remove it.
if wtd_child is not None:
logging.info("Disk %s exists. Checking LUN attachments..." %
wtd_child.DevicePath)
lun = _get_lun_by_wtd(connection, wtd_child)
if lun is not None:
# The lun exists. Delete the mapping before proceeding
logging.info(
"Disk %s id attached to lun %s. I need to remove this lun/mapping."
% (wtd_child.DevicePath, lun.TargetIQN))
# TODO: shutdown all the connections?
lun.RemoveAllWTDisks()
logging.info("All disks detached from lun %s." % lun.TargetIQN)
# Now that the lun is disconnected, delete the WT_DISK
logging.info("Deleting disk %s..." % wtd_child.DevicePath)
files = wtd_child.DevicePath
wtd_child.Delete_()
# Also delete the files from the disk
vhdfiles = conn_cimv2.query(
"Select * from CIM_DataFile where Name = '" + files + "'")
if len(vhdfiles) > 0:
vhdfiles[0].Delete()
# At this point we can proceed by creating a new disk and attaching it to the lun
delta_disk = connection.WT_Disk.NewDiffWTDisk(
ParentPath=parent_vhdx_path, DevicePath=child_vhdx_path)[0]
logging.info("Differencing disk created.")
# Make sure the lun exists and is correctly configured
host = _get_or_create_target(connection, iscsi_target_name)
if iqn is not None:
host.TargetIQN = iqn
logging.info("ISCSI target created/configured %s <-> %s." %
(host.HostName, host.TargetIQN))
# Now attach that disk to the lun
host.AddWTDisk(delta_disk.WTD)
logging.info("ISCSI disk %s attached to lun %s %s." %
(delta_disk.Devicepath, host.HostName, host.TargetIQN))
# Allow to the configured mac to attach to this target:
id = connection.WT_IDMethod.SpawnInstance_()
id.HostName = host.HostName
id.Method = 3
id.Value = str(initiator_mac).replace(":", "-")
id.Put_()
return host.TargetIQN
def handle(self):
"""
This method implements the main logic of the sewrver. It basically routes all the incoming requests to
specific sub-methods, each one in charge of a particular message type. Every message, indeed, must contain the
"command" field, which identifies the action to be taken by the server.
Note that no integrity/identity check is performed at this stage. Future implementations might take this part
into account and implement security in some way.
:return:
"""
try:
client_address = ":".join(
[str(x) for x in self.request.getpeername()])
app_log.info("Connection from %s" % client_address)
# Read the first message
msg = self._read_message()
app_log.debug("Received message from %s: %s" %
(client_address, msg))
# Check basic message format: at least each message should include MAC and COMMAND
if 'command' not in msg or 'mac' not in msg:
raise self.ProtocolException(
"The message read from the socket is wrong: command or mac missing."
)
if 'mac' not in msg:
raise self.ProtocolException(
"Protocol error. The REPORT_WORK request did not contain any mac attribute."
)
if not validate_mac(msg['mac']):
raise self.ProtocolException(
"Protocol error. REPORT_WORK request contained an invalid mac (%s)."
% msg['mac'])
mac = MacAddress.MacAddress(msg['mac'])
# Make sure the client belongs to the ones we administrate.
machine = self.server._app_manager.lookup_by_mac(mac)
if not self.server._app_manager._manager_disabled and machine is None:
raise self.ProtocolException(
"Given mac %s does not match with any machine handled by this manager."
% mac)
# GET_WORK: client is ready to receive a job
if msg['command'] == "GET_WORK":
mac, job = self._handle_get_work(msg)
if job is None:
app_log.info(
"Machine %s has no work to do. Shutting it down." %
machine)
#machine.get_manager().stop_guest(machine)
machine.get_manager().revert_guest(machine)
machine.get_manager().stop_guest(machine)
# REPORT_WORK: client has done its job and wants to send back the report
elif msg['command'] == "REPORT_WORK":
work_id, dest, network_conf = self._handle_report_work(msg)
# If the machine is done and it was a VM, we need to revert it!
self.server._app_manager.notify_done(mac,
work_id,
dest,
network_conf=network_conf,
error=False)
else:
# This is a logic error/unexpected message type
raise self.ProtocolException(
"Protocol error. Received message %s from host %s" %
(msg, client_address))
except Exception as e:
app_log.exception("Error occurred.")
finally:
# Always release socket resources.
self.request.close()
def register_worker(self, machine_mac, worker_status):
with self._lock:
db.jobmanager.register_worker(
host_controller_id=CFG.host_controller_id,
mac_address=str(MacAddress.MacAddress(machine_mac)),
worker_status=worker_status)
def create_guest(self):
"""
Creates a Virtual Machine with a differencing disk based on an immutable image.
The machine is not started.
:return: IMachine
"""
ctx = self.getMgr()
name = str(uuid.uuid4())
diff_path = path.join(self._diff_disk_dir, name + '.vmdk')
logging.info("Creating VM %s" % name)
# Create and register a new machine
# TODO parametrize ostype?
ostype = 'Windows7'
m = ctx.vbox.createMachine(
'', # Settings file
name, # Name
[self._vm_group], # groups
ostype, # OS Type
'forceOverwrite=0' # flags
)
ctx.vbox.registerMachine(m)
logging.debug("VM %s created and registered" % name)
# Set up created machine
session = ctx.mgr.getSessionObject(ctx.vbox)
try:
m.lockMachine(session, ctx.mgr.constants.LockType_Write)
mutable = session.machine
# CPU, ram, vram
logging.debug("Configuring VM %s" % name)
mutable.CPUCount = int(self._conf['cpu_count'])
mutable.memorySize = int(self._conf['memory_size'])
mutable.VRAMSize = int(self._conf['vram_size'])
mutable.accelerate3DEnabled = int(
self._conf['accelerate_3d_enabled'])
mutable.accelerate2DVideoEnabled = int(
self._conf['accelerate_2d_video_enabled'])
eth0 = mutable.getNetworkAdapter(0)
eth0.adapterType = ctx.mgr.constants.all_values(
'NetworkAdapterType')[str(self._conf['adapter_intranet_type'])]
eth0.attachmentType = ctx.mgr.constants.all_values(
'NetworkAttachmentType')[str(
self._conf['adapter_intranet_attachment'])]
eth0.internalNetwork = CFG.vbox_intranet_network_name
eth0.enabled = True
logging.debug("Creating diff VHD for VM %s in %s" %
(name, diff_path))
# Storage: create differential disk and attach it to the new machine.
# The diff disk will be created with autoreset = True, so next time we don't need to
# make this operation again.
base = ctx.vbox.openMedium(self._base_disk_location,
ctx.mgr.constants.DeviceType_HardDisk,
ctx.mgr.constants.AccessMode_ReadOnly,
False)
medium = ctx.vbox.createMedium(
'vmdk', # format
diff_path, # location
ctx.mgr.constants.AccessMode_ReadWrite,
ctx.mgr.constants.DeviceType_HardDisk)
p = base.createDiffStorage(medium,
[ctx.mgr.constants.MediumVariant_Diff])
p.waitForCompletion(DEFAULT_OP_TIMEOUT)
medium.autoReset = True
sata_controller = mutable.addStorageController(
'sata_disk', ctx.mgr.constants.StorageBus_SATA)
# Limit the maximum number of SATA ports for this controller in order to reduce boot time
sata_controller.portCount = 1
mutable.attachDevice('sata_disk', 0, 0,
ctx.mgr.constants.DeviceType_HardDisk, medium)
mutable.BIOSSettings.IOAPICEnabled = True
logging.debug("Saving settings for VM %s" % name)
mutable.saveSettings()
mac = str(m.getNetworkAdapter(0).MACAddress)
# At this point everything went ok, add the machien to the list of managed machines
tmp = VBoxMachine(manager=self, mac=mac, id=m.id)
logging.info("Machine %s successfully created" % name)
if self._status_handler is not None:
self._status_handler.register_worker(
machine_mac=MacAddress.MacAddress(mac),
worker_status=WorkerStatus.IDLE)
return tmp # type: VBoxMachine
except Exception as e:
logging.exception("Error occurred during vm configuration %s" %
name)
# Exception occurred. Discard any change and raise again.
try:
if session.state == ctx.mgr.constants.SessionState_Locked:
session.unlockMachine()
except Exception: # varoius errors (xpcom.Exception)
pass
media = m.unregister(
ctx.mgr.constants.all_values('CleanupMode')
['DetachAllReturnHardDisksOnly'])
# Delete attached mediums
for medium in media:
medium.deleteStorage()
# No matter if you can recover, raise again.
raise
finally:
try:
if session.state == ctx.mgr.constants.SessionState_Locked:
session.unlockMachine()
except Exception: # varoius errors (xpcom.Exception)
pass
def _handle_report_work(
self,
msg # type:dict
):
"""
Handles the REPORT_WORK message type. In particular, this method collects report and network context from the
guest machine and stores it into the central DB.
:param msg:
:return:
"""
client_address = ":".join([str(x) for x in self.request.getpeername()])
cur_thread = threading.current_thread().getName()
app_log.info("%s: Handling REPORT_WORK request from %s" %
(cur_thread, client_address))
exp_id = None
# Parse all the info within the message from client
if 'mac' not in msg:
raise self.ProtocolException(
"Protocol error. The REPORT_WORK request did not contain any mac attribute."
)
if not validate_mac(msg['mac']):
raise self.ProtocolException(
"Protocol error. REPORT_WORK request contained an invalid mac (%s)."
% msg['mac'])
mac = MacAddress.MacAddress(msg['mac'])
if 'status' not in msg:
raise self.ProtocolException(
"Protocol error. The REPORT_WORK request did not contain any status attribute."
)
if 'report_bytes_len' not in msg:
raise self.ProtocolException(
"Protocol error. The REPORT_WORK request did not contain any report_bytes_len attribute."
)
if 'network_conf' not in msg:
raise self.ProtocolException(
"Protocol error. The REPORT_WORK request did not contain any network_conf attribute."
)
length = 0
try:
length = int(msg['report_bytes_len'])
except ValueError:
raise self.ProtocolException(
"Protocol error. REPORT_WORK request contained an invalid report_bytes_len (%s)."
% msg['report_bytes_len'])
if length < 0:
raise self.ProtocolException(
"Protocol error. REPORT_WORK request contained a negative report_bytes_len (%d)."
% length)
work_id = -1
if 'work_id' not in msg:
raise self.ProtocolException(
"Protocol error. The REPORT_WORK request did not contain any work_id attribute."
)
try:
work_id = int(msg['work_id'])
exp_id = db.lookup_experiment_id_by_work_id(work_id)
if exp_id is None:
raise self.ProtocolException(
"The worker id provided did not match any experiment.")
except ValueError:
raise self.ProtocolException(
"Protocol error. REPORT_WORK request contained an invalid work_id (%s)."
% msg['work_id'])
self.server._app_manager._machine_status_handler.notify_machine_status(
machine_mac=mac, worker_status=WorkerStatus.REPORTING)
# The client also has to provide its current network context. We need its IP and its DEFAULT_GW. We will also
# add info about
network_conf = msg['network_conf']
if not validate_network_conf(network_conf):
raise self.ProtocolException(
"Protocol error. Invalid or incomplete network_conf json string: %s."
% msg['network_conf'])
if not self.server._app_manager._manager_disabled:
# Now the client expects a sort of ACK to start report transmission. Before giving this ACK, we want to stop
# the associated sniffer, so that report transmission does not impact on collected traffic.
app_log.info("%s: Stopping sniffer for machine with mac %s" %
(cur_thread, mac))
machine = self.server._app_manager.lookup_by_mac(mac)
machine.get_manager().stop_network_sniffing(machine)
answer = {'response': 'REPORT_WORK_RESP'}
self._write_message(answer)
# Receive the file
dest = build_output_report_fullpath(exp_id)
app_log.info("%s: REPORT_WORK request from %s - Receiving file to %s" %
(cur_thread, client_address, dest))
self.__recv_file(destinationpath=dest, filelen=length)
app_log.info("%s: REPORT_WORK request from %s - Received file to %s" %
(cur_thread, client_address, dest))
# Ok, let the client know we are done with file transfer
answer = {'response': 'REPORT_WORK_REPORT_RECEIVED'}
self._write_message(answer)
app_log.info("%s: REPORT_WORK request from %s HANDLED OK" %
(cur_thread, client_address))
# Done!
return work_id, dest, network_conf
def _handle_get_work(
self,
msg # type: dict
):
"""
This method handles the request GET_WORK. It takes care of polling the db for a job, serve it to the client,
and start the network sniffing.
:param msg:
:return:
"""
client_address = ":".join([str(x) for x in self.request.getpeername()])
cur_thread = threading.current_thread().getName()
app_log.info("%s: Handling GET_WORK request from %s" %
(cur_thread, client_address))
# Parse all the info within the message from client
if 'mac' not in msg:
raise self.ProtocolException(
"Protocol error. The GET_WORK request did not contain any mac attribute."
)
if not validate_mac(msg['mac']):
raise self.ProtocolException(
"Protocol error. GET_WORK request contained an invalid mac (%s)."
% msg['mac'])
mac = MacAddress.MacAddress(msg['mac'])
# Let's pop a job from the db and then send data to the client
mgr = db.jobmanager
id, path = mgr.get_work(mac)
# If get_work() returns a NONE ID, it means we have nothing more to do at the moment.
if id is None:
app_log.info("No work for client %s" % client_address)
return mac, None
# Otherwise we got a valid job ID to be processed.
# Convert the obtained path into a locally valid path
path = db.translate_installer_path(path)
file_dim = 0
file_name = None
# Given the path, extract the name and the dimension of the file. We assume the file exists.
file_name = os.path.basename(path)
file_dim = os.path.getsize(path)
app_log.info("%s: Sending work id %d (%s) to client %s" %
(cur_thread, id, file_name, client_address))
response = {
'response': 'GET_WORK_RESP',
'work_id': id,
'file_name': file_name,
'file_dim': file_dim
}
# Send the get_work_response to client
self._write_message(response)
# Wait for GET_WORK_FILE message...
answer = self._read_message()
if 'command' not in answer:
raise self.ProtocolException(
"Message from the client does not contain response command.")
if answer['command'] != 'GET_WORK_FILE':
raise self.ProtocolException(
"Unexpected command received by client: %s, expecting GET_WORK_FILE"
% answer['command'])
# Send the binary to analyze
self.__send_file(path)
# Wait for client's WORK_FILE_RECEIVED message.
answer = self._read_message()
if 'command' not in answer:
raise self.ProtocolException(
"Message from the client does not contain response command.")
if answer['command'] != 'GET_WORK_FILE_RECEIVED':
raise self.ProtocolException(
"Unexpected command received by client: %s, expecting GET_WORK_FILE_RECEIVED"
% answer['command'])
# Start the sniffer addociated to this guest
if not self.server._app_manager._manager_disabled:
app_log.info("%s: Starting sniffer for machine with mac %s" %
(cur_thread, mac))
machine = self.server._app_manager.lookup_by_mac(mac)
machine.get_manager().start_network_sniffing(machine)
response = {'response': 'GET_WORK_START'}
# Now let the guest agent start the work
self._write_message(response)
self.server._app_manager._machine_status_handler.notify_machine_status(
machine_mac=mac, worker_status=WorkerStatus.ANALYZING)
# Done!
return mac, id