def pingable(self):
if not self.enabled():
L.verbose("{} not enabled".format(self.name()))
return False
ips = self.ip_addresses()
if not ips:
L.verbose("{} has no IP address".format(self.name()))
return False
cmd = ['ping', '-n', '1', '-w', '2', '-S', ips[0].exploded, '8.8.8.8']
try:
output = check_subprocess(cmd)[0]
if 'Received = 1' in output:
return True
else:
# Consider this a real error and propagate. It's likely a code issue.
raise XVEx(
"Don't know how to parse ping output: {}".format(output))
except XVProcessException as ex:
if 'Lost = 1' in ex.stderr:
L.debug("Couldn't ping on adapter {}".format(self))
return False
L.warning("Ping failed unexpectedly with error '{}'. "
"Assuming adapter {} un-pingable.".format(
ex.stderr, self))
return False
def _iter_package(package, restrict_platform=True, is_pkg=True):
def ignore_package(package, ignore_substrings):
# Don't import anything which doesn't belong on this OS
for ignore in ignore_substrings:
if ignore in package:
return True
return False
if restrict_platform and ignore_package(package, other_oses()):
return
try:
L.debug("Importing {}".format(package))
mod = importlib.import_module(package)
package_path = os.path.normpath(os.path.split(mod.__file__)[0])
yield mod
except ImportError as ex:
raise XVEx(
"Can't import package '{}' ({}). Make sure the package directory is in your "
"sys.path".format(package, ex))
if not is_pkg:
return
L.verbose("Walking package path {}".format(package_path))
for _, module_name, is_pkg2 in pkgutil.walk_packages([package_path]):
for mod in _iter_package("{}.{}".format(package, module_name),
restrict_platform, is_pkg2):
yield mod
def push(self, src, dst):
self._ensure_connected()
# TODO: Can this do dirs?
L.verbose("Pushing file/directory to remote machine: {} -> {}".format(
src, dst))
self._sftp_client.put(src, dst)
def _rewrite_root_rules(rules):
L.verbose("Writing new pfctl rules: {}".format(rules))
rules_file = PFCtl._create_rules_file(rules)
PFCtl._pfctl(['-Fr'])
PFCtl._pfctl(['-f', rules_file])
L.debug("Rewrote root pfctl rules")
def write_remote_file_from_contents(self, path, contents):
dst_filename = os.path.split(path)[1]
hfile, tmpfile = tempfile.mkstemp("_{}".format(dst_filename),
"xv_leak_test_")
with os.fdopen(hfile, "w") as file_:
file_.write(contents)
L.verbose("Writing remote file {}".format(path))
self._device.connector().push(tmpfile, path)
def _getinfo(self):
cmd = ['networksetup', '-getinfo', self._name]
lines = check_subprocess(cmd)[0].splitlines()
L.verbose("{} returned:\n{}".format(' '.join(cmd), lines))
info = {}
for line in lines:
match = NetworkService.PROG_INFO_LINE.match(line)
if not match:
continue
info[match.group(1).strip()] = match.group(2).strip()
return info
def execute_command(self, cmd, root=False):
cmd = ConnectorHelper._sanitize_cmd(cmd)
# Escape spaces in paths. First part of command must be the executable.
cmd[0] = "\"{}\"".format(cmd[0])
script = '''\
cd {}
{}'''.format(self._device.tools_root(), ' '.join(cmd))
L.debug("Executing shell command: {}".format(' '.join(cmd)))
L.verbose("Using bash wrapper script:\n{}".format(script))
return self.execute_script_remotely(script, root)
def execute_subprocess(cmd):
L.verbose("execute_subprocess: Executing command: {}".format(
' '.join(cmd)))
process = subprocess.Popen(cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
retcode = process.wait()
try:
return retcode, stdout.decode(), stderr.decode()
except UnicodeDecodeError:
return retcode, stdout.decode('cp1252'), stderr.decode('cp1252')
def _pull_file_append(self, src, dst):
L.verbose("Appending file from remote machine to local file: {} <- {}".format(dst, src))
hfile, temp_path = tempfile.mkstemp(
suffix="_{}".format(os.path.split(src)[1]), prefix='xv_leak_test_')
os.close(hfile)
self._sftp_client.get(src, temp_path)
with open(temp_path) as f_src:
contents = f_src.read()
with open(dst, "a") as f_dst:
f_dst.write(contents)
def command_line_for_pid(self, pid):
if self.is_cygwin():
# The psutil module isn't supported on cygwin
cmd = [
"wmic", "process", "where", "ProcessID='{}'".format(pid),
"get", "CommandLine"
]
args = self._connector_helper.check_command(cmd)[0]
L.verbose("Raw wmic command line was: {}".format(args))
return WindowsDevice._fix_quotes(args)
return self._connector_helper.execute_scriptlet(
'command_line_for_pid.py', [pid])
def lookup(self, hostname, timeout, server=None):
# dig doesn't like floats for timeout
timeout = int(math.ceil(timeout))
if server:
cmd = [
'dig', "+time={}".format(timeout), hostname,
"@{}".format(server)
]
else:
cmd = ['dig', "+time={}".format(timeout), hostname]
stdout = self._execute(cmd)[0]
L.verbose("dig output: {}".format(stdout))
return Dig._parse_output(stdout)
def execute_python(self, cmd, root=False):
cmd = ConnectorHelper._sanitize_cmd(cmd)
# Escape spaces in paths. First part of command must (currently) be the script. Potentially
# we might support flags to python I guess.
cmd[0] = "\"{}\"".format(cmd[0])
script = '''\
set -e
cd {}
source activate
set +e
python {}'''.format(self._device.tools_root(), ' '.join(cmd))
L.debug("Executing python cmd: {}".format(' '.join(cmd)))
L.verbose("Using python wrapper script:\n{}".format(script))
return self.execute_script_remotely(script, root)
def _build_all_components_for_device(self, device, configs):
for component_name in self._component_factory.builder_names():
try:
device[component_name] = self._component_factory.build(
component_name, device, configs.get(component_name, {}))
if device[component_name] is None:
raise XVEx(
"Component factory returned None for component type {} {}"
.format(device, component_name))
except ComponentNotSupported as ex:
# It's perfectly fine for building to fail. Some components can't be built on
# certain devices. However, if any other form of exception occurs then propagate it
# so we don't mask unexpected issues.
L.verbose("Can't build component {} for device {}: {}".format(
component_name, device.device_id(), ex))
def wmic_rows():
rows = []
nic_rows = parse_wmic_output(check_subprocess(['wmic', 'nic'])[0])
nicconfig_rows = parse_wmic_output(
check_subprocess(['wmic', 'nicconfig'])[0])
# We're effectively performing a join on SettingID and GUID here.
for nic_row in nic_rows:
if nic_row['GUID'] == "":
L.verbose("Network adapter '{}' has no GUID. Ignoring it!".format(
nic_row['Name']))
continue
for nicconfig_row in nicconfig_rows:
if nicconfig_row['SettingID'] == nic_row['GUID']:
rows.append(merge_two_dicts(nic_row, nicconfig_row))
break
return rows
def execute_script_remotely(self, script_contents, root=False):
# We generate a collision free (hopefully) name here as doing it on the remote device
# means executing a script, so we get in a loop. We could do that by executing a
# simpler script but this would likely mean it's not cross platform and/or we have
# to worry about escape characters when we pass the script.
hsh = hashlib.md5()
hsh.update(script_contents.encode("utf-8"))
remote_script_filename = "{}.sh".format(hsh.hexdigest())
script_dst = os.path.join(self._device.temp_directory(),
remote_script_filename)
self.write_remote_file_from_contents(script_dst, script_contents)
L.verbose("Executing remote script {}".format(script_dst))
return self._device.connector().execute(['bash', script_dst],
root=root)
def __init__(self, device, parameters):
super().__init__(device, parameters)
self._restrict_parameters(must_disrupt=True, must_restore=False)
# TODO: Does this really exclude all adapters we don't want? Maybe exclude TAP somehow?
adapters = self._device['network_tool'].adapters_in_priority_order()
L.verbose("All adapters: {}".format(adapters))
adapters = [adapter for adapter in adapters if adapter.pingable()]
L.verbose("Pingable adapters: {}".format(adapters))
if len(adapters) < 2:
raise XVEx(
"There must be at least 2 pingable adapters. All pingable adapters are {}"
.format(adapters))
self._adapter1 = adapters[0]
self._adapter2 = adapters[1]
self._adapter1_original_metric = self._adapter1.interface_metric()
self._adapter2_original_metric = self._adapter2.interface_metric()
def _pull_dir(self, src, dst, append_duplicate=True):
L.debug("Pulling directory from remote machine: {} <- {}".format(dst, src))
self._sftp_client.chdir(src)
makedirs_safe(dst)
for root, _, files in self._sftp_walk(src):
subdir = os.path.join(dst, os.path.relpath(root, src))
L.verbose("Creating subfolder {}".format(subdir))
makedirs_safe(subdir)
for file_ in files:
src_file = os.path.join(root, file_).replace('\\', '/')
dst_file = os.path.join(subdir, file_).replace('\\', '/')
if append_duplicate and os.path.exists(dst_file):
self._pull_file_append(src_file, dst_file)
else:
L.verbose("Pulling file from remote machine: {} <- {}".format(dst, src))
self._sftp_client.get(src_file, dst_file)
def push(self, src, dst):
ret, stdout, stderr = execute_subprocess(['cp', '-rf', src, dst])
if ret != 0:
raise XVEx("Couldn't copy {} -> {}: stdout: {}, stderr: {}".format(
src, dst, stdout, stderr))
if os.geteuid() != 0:
return
# When executing locally it's entirely likely we'll be running as root. We don't want to
# have any files owned by root unless directly specified by the user - in which case they
# should chown via a command for now (which I think is an unlikely use case).
L.verbose("Removing root permissions from file {} ({})".format(
dst,
tools_user()[0]))
os.chown(dst, tools_user()[0], -1)
def known_servers(self):
output = self._connector_helper.check_command([
'wmic.exe', 'nicconfig', 'where', '"IPEnabled = True"', 'get',
'DNSServerSearchOrder', '/format:rawxml'
])[0]
L.verbose("Got raw wmic output: {}".format(output))
dns_servers = []
for nic in fromstring(output).findall("./RESULTS/CIM/INSTANCE"):
for prop in nic:
if prop.tag != 'PROPERTY.ARRAY':
continue
for val in prop.findall("./VALUE.ARRAY/VALUE"):
ip = ipaddress.ip_address(val.text)
dns_servers.append(ip)
self._check_current_dns_server_is_known(dns_servers)
return dns_servers
def execute(self, cmd, root=False):
self._ensure_connected()
if root and self._username.decode() != 'root' and self._platform() != 'cygwin':
cmd = ['sudo', '-n'] + cmd
cmd = ["[", "-f", ".source", "]", "&&", ".", ".source;"] + cmd
L.verbose("SimpleSSHConnector: Executing remote command: '{}'".format(cmd))
# TODO: This will block if stdin on the remote machine blocks. Can't ctrl-c. Consider a
# custom wrapper with select and timeout
# get_pty is required for accessing environment variables.
output = self._ssh_client.exec_command(' '.join(cmd))
return_code = output[1].channel.recv_exit_status()
stdout = output[1].read().decode().strip()
stderr = output[2].read().decode().strip()
return return_code, stdout, stderr
def execute_scriptlet(self, scriptlet, args, root=False):
'''A scriptlet is a specialised subprocess for executing python code. The code can only do
one of two things: return a python object on success or throw an exception. A scriptlet
must not do any outputting to stdout or stderr which it does not want to be un-pickled.
The ideal use case is executing a python function remotely and getting its return value.
'''
cmd = [
os.path.join(self._device.tools_root(), 'xv_leak_tools',
'scriptlets', scriptlet)
]
ret, stdout, stderr = self.execute_python(cmd + args, root)
L.verbose(
"Scriptlet returned: ret: {}\nstdout: '{}'\nstderr: '{}'".format(
ret, stdout, stderr))
if ret != 0:
raise pickle.loads(codecs.decode(stderr.encode(), "base64"))
else:
return pickle.loads(codecs.decode(stdout.encode(), "base64"))
def lookup(self, hostname, timeout, server=None):
# dig doesn't like floats for timeout
timeout = int(math.ceil(timeout))
if server:
cmd = [
'dig', "+time={}".format(timeout), hostname,
"@{}".format(server)
]
else:
cmd = ['dig', "+time={}".format(timeout), hostname]
# Prevent the output from being empty
stdout = None
while not stdout:
stdout = self._execute(cmd)[0]
if not stdout:
L.verbose("dig output was empty; doing another lookup.")
L.verbose("dig output: {}".format(stdout))
return Dig._parse_output(stdout)
def _wait_for_vm_power(vmx_path):
# Hacky trick to make sure VM has power
timeup = TimeUp(60)
while not timeup:
try:
if VMWareDeviceDiscoverer._vm_state(vmx_path) != 'running':
time.sleep(0.2)
continue
VMWareDeviceDiscoverer._get_vm_ip(vmx_path)
# If we manage to get the IP then we're powered on. This will be unlikely to happen
break
except XVProcessException as ex:
if 'The virtual machine is not powered on' in ex.stdout:
time.sleep(0.2)
continue
# Any other exception means we're powered on
break
if timeup:
raise XVEx("VM never got power: {}".format(vmx_path))
L.verbose('VM now has power')
def query(self, timeout=10):
L.debug("IPResponder getting IPs for {}".format(self._token.decode()))
timeup = TimeUp(timeout)
first_time_around = True
while not timeup:
try:
if not first_time_around:
L.verbose("Trying to querying IP responder server again")
else:
L.verbose("Querying IP responder server")
first_time_around = False
self._query_sock.sendto(self._token + b'?',
self._server_address)
return self._receive()
except (OSError, XVEx) as ex:
# OSError happens if sendto fails (which probably means no network)
# XVEx happens if there's no response to receive.
L.warning("IP responder query failed: {}".format(ex))
time.sleep(1)
raise XVEx("Couldn't query IP Responder server")
def execute(self, cmd, root=False):
self._ensure_connected()
if root and self._username.decode() != 'root':
cmd = ['sudo', '-n'] + cmd
L.verbose(
"SimpleSSHConnector: Executing remote command: '{}'".format(cmd))
chan = self._ssh_client.get_transport().open_session()
chan.settimeout(2)
# TODO: This will block if stdin on the remote machine blocks. Can't ctrl-c. Consider a
# custom wrapper with select and timeout
chan.exec_command(' '.join(cmd))
bufsize = -1
# stdin = chan.makefile('wb', bufsize)
stdout = chan.makefile('r', bufsize)
stderr = chan.makefile_stderr('r', bufsize)
retcode = chan.recv_exit_status()
# paramiko splits output into a list AND keeps the newline. So joining like this puts it
# back to how we'd "normally" expect output.
return retcode, ''.join(stdout), ''.join(stderr)
def _curl_url(self, url):
L.debug("Curl-ing url {} to find ips".format(url))
# TODO: Can't use pycurl on cygwin. Need to figure out how to compile cleanly.
# try:
# L.debug("curl-ing {}".format(url))
# curl_instance = curl.Curl(url)
# curl_instance.set_option(pycurl.TIMEOUT, 5)
# response = curl_instance.get()
# print response
# except pycurl.error as ex:
# errno, message = ex.args
# if errno == pycurl.E_COULDNT_CONNECT:
# L.warning(
# "pycurl couldn't connect to {}. Assuming no public IP available.".format(url))
# return None
# # Be cautious. Haven't assessed what other errors are acceptable so let's
# # just raise up
# # the error.
# raise
try:
cmd = ['curl', url, '--connect-timeout', '5']
if self._max_time:
cmd += ['-m', self._max_time]
return self._execute(cmd)
except XVProcessException as ex:
for error in IPToolCurl.ACCEPTABLE_CURL_ERRORS:
if error in ex.stderr:
L.verbose(
"curl couldn't connect to {}. Assuming no public IP available."
.format(url))
return "", ""
# Be cautious. Haven't assessed what other errors are acceptable so let's just raise up
# the error.
raise
def stop(self):
L.info('Stopping packet capture and getting packets')
packets = []
for capturer in self._capturers:
packets += capturer.stop()
# TODO: Fix these issues in the capturer itself?
packets = [packet for packet in packets if packet['DestIP']]
packets = [
packet for packet in packets
if not is_mac_address(packet['DestIP'])
]
for packet in packets:
packet['SourceIP'] = ipaddress.ip_address(packet['SourceIP'])
packet['DestIP'] = ipaddress.ip_address(packet['DestIP'])
if not packets:
L.warning(
'No packets were captured. This is probably not what you want.'
)
else:
L.info('{} packets captured'.format(len(packets)))
L.verbose("List of all packets:\n{}".format(Packets(packets)))
return packets
def import_all_from_package(package, restrict_platform=True):
for mod in _iter_package(package, restrict_platform):
L.verbose("Imported {}".format(mod.__file__))
def execute_subprocess(cmd):
L.verbose("execute_subprocess: Executing command: {}".format(' '.join(cmd)))
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
stdout, stderr = process.communicate()
retcode = process.wait()
return retcode, stdout.decode('utf-8'), stderr.decode('utf-8')