def wait_for(self, what, timeout):
method = getattr(self, what)
start = time.time()
okay = method()
while not okay and time.time() - start < timeout:
time.sleep(1)
okay = method()
if not okay:
raise XVEx("Network service {} never became {}".format(self, what))
L.debug("Network service {} became {}".format(self, what))
def __del__(self):
if self._no_disable:
return
self._ensure_anchor_is_removed()
L.debug("Releasing pfctl reference {}".format(self._token))
try:
self._pfctl(['-X', self._token])
except XVProcessException as ex:
# This can happen as some providers will aggressively tear down the firewall when they
# disconnect (using pfctl -d) and thus remove our reference (token) to the firewall.
L.warning("Failed to terminate pf firewall: {}".format(ex))
def _command_line_args(self):
if self._command_line_args_cache:
return self._command_line_args_cache
# This can throw. Let it! If we know we're using openvpn but can't get the command line then
# that's a bug.
self._command_line_args_cache = self._device.command_line_for_pid(
self._openvpn_process())
L.debug("Got command line args: {}".format(
self._command_line_args_cache))
return self._command_line_args_cache
def open_app(self, app_path, root=False): # pylint: disable=unused-argument
# TODO: Oh dear god. This was painful!
# Note that this can fail in a bad way. If start fails then it will pop a dialog box and
# freeze everything. Need a better solution to this. Make sure paths are correct for now!
head, tail = windows_path_split(app_path)
cmd = [
'cmd', '/C', 'start', '/D',
"\"{}\"".format(windows_real_path(head)), tail
]
L.debug("Executing cmd '{}'".format(cmd))
self._connector_helper.check_command(cmd)
def _ssh_disconnect(self):
L.debug("Disconnecting SSH")
if self._sftp_client:
self._sftp_client.close()
self._sftp_client = None
if self._ssh_client:
self._ssh_client.close()
self._ssh_client = None
self._remove_routes()
def _delete_testing_chain(self):
# Flush all rules in our chain
self._connector_helper.check_command(
["iptables", "-w", "-F", self._testing_chain_name], root=True)
# Remove all references (jumps) to our chain
L.debug("Cleaning up iptables testing chain")
for source_chain in ["INPUT", "OUTPUT"]:
self._delete_rule(self._jump_rule_args(source_chain))
self._delete_chain(self._testing_chain_name)
def set_network_service_order(services):
L.debug("Setting network service order: {}".format(services))
preferences = _MacOSNetworkHelper.create_preferences()
current_set = SCNetworkSetCopyCurrent(preferences)
service_ids = [service.id() for service in services]
if not SCNetworkSetSetServiceOrder(current_set, service_ids) or \
not _MacOSNetworkHelper.save_preferences(preferences):
raise XVEx(
"Couldn't set network service order for location {}".format(
SCNetworkSetGetName(current_set)))
def _vpn_server_ip(self):
if not self._command_line_args():
L.debug("Couldn't get command line args so can't determine the VPN server IP")
return None
ip = self._try_vpn_server_ip_from_command_line()
if ip:
return ipaddress.ip_address(ip)
ip = self._try_vpn_server_ip_from_config_file()
if ip:
return ipaddress.ip_address(ip)
return None
def lookup(self, hostname=None, timeout=None, server=None):
if hostname is None:
hostname = DNSLookupTool.DEFAULT_HOSTNAME
if timeout is None:
timeout = DNSLookupTool.DEFAULT_TIMEOUT
L.debug("Doing DNS lookup for '{}'".format(hostname))
server, ips = self._dig.lookup(hostname, timeout, server)
L.debug("DNS lookup returned ips: {} using server {}".format(
ips, server))
return server, ips
def execute_command(self, cmd, root=False):
cmd = ConnectorHelper._sanitize_cmd(cmd)
# Escape spaces in paths. First part of command must be the executable.
cmd[0] = "\"{}\"".format(cmd[0])
script = '''\
cd {}
{}'''.format(self._device.tools_root(), ' '.join(cmd))
L.debug("Executing shell command: {}".format(' '.join(cmd)))
L.verbose("Using bash wrapper script:\n{}".format(script))
return self.execute_script_remotely(script, root)
def _network_location_info(self):
L.debug("Getting network location info")
ret = ''
ret += 'Network Locations\n'
ret += '-----------------\n\n'
location_data = []
for location in self._macos_network.network_locations():
data = ''
data += "Name: {}\n".format(location.name())
location_data.append(data)
ret += '\n'.join(location_data)
ret += '\n'
return ret
def start(self):
L.debug('Starting IPResponder thread')
if self._server_address is None:
raise XVEx(
"Server address must be set before starting IPResponder")
if self._send_forever_thread is not None:
raise XVEx("IP Responder already started")
self._check_server()
self._stop_sending = threading.Event()
self._send_forever_thread = threading.Thread(target=self._send_forever)
self._send_forever_thread.start()
def stop(self):
L.debug("Stopping packet capture on interface {} and getting packets".
format(self._interface))
if not self._pid_file:
raise XVEx("Packet capture not started!")
# TODO: Switch to pid file kill
cmd = ['killall', '-SIGINT', 'xv_packet_capture']
self._connector_helper.check_command(cmd, root=True)
self._pid_file = None
return self._get_packets()
def stop(self):
L.debug('Stopping IPResponder thread')
if self._send_forever_thread is None:
return
self._stop_sending.set()
L.debug('Waiting for thread to die...')
while self._send_forever_thread.is_alive():
continue
self._stop_sending = None
self._send_forever_thread = None
def _kill_server(self):
if self._server_running:
L.debug("Killing Selenium server")
selenium_process = self._device.connector().execute(
['lsof', '-t', '-i', ':4444'])
if not selenium_process[0]:
selenium_pid = selenium_process[1].strip()
self._device.connector().execute(['kill', selenium_pid])
self._server_running = False
else:
L.warning(
"Couldn't find the Selenium server process; try killing Java"
)
def add_torrent(self, magnet_uri):
if not self._daemon_started:
L.debug("Transmission daemon not started; starting...")
self.open()
# TODO: replace this sleep with a timeout.
time.sleep(1)
L.debug("Adding {} to Transmission".format(magnet_uri))
cmd = ['transmission-remote', '--add', "{}".format(magnet_uri)]
out = self._device.connector().execute(cmd)
if out[0]:
raise XVProcessException(cmd, *out)
self.added_torrents.append(self._object_from_magnet_uri(magnet_uri))
def ensure_leak_test_anchor_present():
rules = PFCtl._read_root_rules()
for rule in rules:
if PFCtl.LEAK_TEST_ANCHOR in rule:
L.debug("Leak test anchor {} already present in rules".format(
PFCtl.LEAK_TEST_ANCHOR))
return
rules.append("anchor \"{}\" all".format(PFCtl.LEAK_TEST_ANCHOR))
rules_file = PFCtl._create_rules_file(rules)
PFCtl._pfctl(['-Fr'])
PFCtl._pfctl(['-f', rules_file])
L.debug("Rewrote root pfctl rules")
def dns_server_ips(self):
if not self._tap_adapter_name:
L.debug(
"Don't know TAP adapter name for VPN application {}. Falling back to default "
"method to get DNS server IPs".format(self._config["name"]))
return super().dns_server_ips()
adapter = self._device['network_tool'].adapter_by_name(self._tap_adapter_name)
if adapter:
return adapter.dns_servers()
L.warning(
"Couldn't find TAP adapter '{}' for VPN application '{}'. Falling back to default "
"method to get DNS server IPs".format(self._tap_adapter_name, self._config["name"]))
return super().dns_server_ips()
def execute_python(self, cmd, root=False):
cmd = ConnectorHelper._sanitize_cmd(cmd)
# Escape spaces in paths. First part of command must (currently) be the script. Potentially
# we might support flags to python I guess.
cmd[0] = "\"{}\"".format(cmd[0])
script = '''\
set -e
cd {}
source activate
set +e
python {}'''.format(self._device.tools_root(), ' '.join(cmd))
L.debug("Executing python cmd: {}".format(' '.join(cmd)))
L.verbose("Using python wrapper script:\n{}".format(script))
return self.execute_script_remotely(script, root)
def detect(self):
if current_os() != "macos":
return None
L.debug("Trying to determine if we're using a macOS network extension")
vpn_info = VPNInfo()
if not self._ne_processes():
L.debug('Not using a network extension')
return None
L.info("Detected a VPN network extension (unknown protocol)")
vpn_info.vpn_processes = self._ne_processes()
return vpn_info
def _ensure_anchor_is_present(self):
rules = PFCtl._read_root_rules()
new_rules = []
for rule in rules:
if "xvpn" in rule:
# Ensure we always put the leak testing rules before xvpn ones.
# TODO: This isn't vendor agnostic. We should make it so.
new_rules.append("anchor \"{}\" all".format(self._anchor_name))
new_rules.append(rule)
if self._anchor_name in rule:
L.debug("Leak test anchor {} already present in rules".format(
self._anchor_name))
return
PFCtl._rewrite_root_rules(new_rules)
def test(self):
L.describe('Get the public IP addresses before VPN connect')
public_ips_before_connect = self.localhost[
'ip_tool'].all_public_ip_addresses()
# Sanity check: We should always have at least one IP address!
self.assertNotEmpty(public_ips_before_connect,
"Couldn't get public IP addresses")
L.info("Public IP addresses before VPN connect are {}".format(
public_ips_before_connect))
L.describe('Open and connect the VPN application')
self.localhost['vpn_application'].open_and_connect()
self._check_network(20)
L.describe('Get the public IP addresses after VPN connect')
public_ips_after_connect = self.localhost[
'ip_tool'].all_public_ip_addresses()
timeout = TimeUp(20)
while not timeout:
public_ips_after_connect = self.localhost[
'ip_tool'].all_public_ip_addresses()
if public_ips_after_connect:
break
L.debug("Waiting {} s for a public IP address".format(
int(timeout.time_left())))
time.sleep(4)
self.assertNotEmpty(
public_ips_after_connect,
"Couldn't get public IP addresses after connecting to the VPN server"
)
L.info("Public IP addresses after VPN connect are {}".format(
public_ips_after_connect))
L.describe(
'Expect public IP addresses after VPN connect to have all changed')
unchanged_ips = set(public_ips_before_connect).intersection(
public_ips_after_connect)
self.assertEmpty(
unchanged_ips,
"These IPs did not get changed after connection to VPN {}".format(
unchanged_ips))
def wmic_rows():
rows = []
nic_rows = parse_wmic_output(check_subprocess(['wmic', 'nic'])[0])
nicconfig_rows = parse_wmic_output(
check_subprocess(['wmic', 'nicconfig'])[0])
# We're effectively performing a join on SettingID and GUID here.
for nic_row in nic_rows:
if nic_row['GUID'] == "":
L.debug("Network adapter '{}' has no GUID. Ignoring it!".format(
nic_row['Name']))
continue
for nicconfig_row in nicconfig_rows:
if nicconfig_row['SettingID'] == nic_row['GUID']:
rows.append(merge_two_dicts(nic_row, nicconfig_row))
break
return rows
def test(self):
L.describe('Open and connect the VPN application')
self.target_device['vpn_application'].open_and_connect()
L.describe('Capture traffic')
self.capture_device['packet_capturer'].start()
L.describe('Generate whatever traffic you want')
message_and_await_enter('Are you done?')
L.describe('Stop capturing traffic')
packets = self.capture_device['packet_capturer'].stop()
whitelist = self.capture_device.local_ips()
L.debug('Excluding {} from analysis'.format(whitelist))
self.traffic_analyser.get_vpn_server_ip(packets, whitelist)
def _swap_highest_priority_services(self):
def active_service_indices(services):
for service in services:
if service.active():
yield services.index(service)
services = self._device['network_tool'].network_services_in_priority_order()
try:
active_service_index = active_service_indices(services)
i = next(active_service_index)
j = next(active_service_index)
L.debug('Swapping {}, {}'.format(services[i], services[j]))
services[i], services[j] = services[j], services[i]
except StopIteration:
raise XVEx('There must be at least two active services')
self._device['network_tool'].set_network_service_order(services)
def _ensure_anchor_is_removed(self):
rules = PFCtl._read_root_rules()
new_rules = []
need_update = False
for rule in rules:
if self._anchor_name not in rule:
new_rules.append(rule)
else:
need_update = True
if not need_update:
L.debug(
"Leak test anchor {} wasn't found. No need to remove".format(
self._anchor_name))
return
PFCtl._rewrite_root_rules(new_rules)
def _pull_dir(self, src, dst, append_duplicate=True):
L.debug("Pulling directory from remote machine: {} <- {}".format(dst, src))
self._sftp_client.chdir(src)
makedirs_safe(dst)
for root, _, files in self._sftp_walk(src):
subdir = os.path.join(dst, os.path.relpath(root, src))
L.verbose("Creating subfolder {}".format(subdir))
makedirs_safe(subdir)
for file_ in files:
src_file = os.path.join(root, file_).replace('\\', '/')
dst_file = os.path.join(subdir, file_).replace('\\', '/')
if append_duplicate and os.path.exists(dst_file):
self._pull_file_append(src_file, dst_file)
else:
L.verbose("Pulling file from remote machine: {} <- {}".format(dst, src))
self._sftp_client.get(src_file, dst_file)
def __init__(self, no_disable=False):
self._no_disable = no_disable
self._token = None
if self._no_disable:
self._pfctl(['-e'])
else:
lines = self._pfctl(['-E'])[1].splitlines()
for line in lines:
match = PFCtl.PROG_TOKEN.match(line)
if not match:
continue
self._token = match.group(1)
L.debug("Got pfctl reference {}".format(self._token))
if not self._token:
raise XVEx(
"Couldn't parse token from pfctl output:\n {}".format(
"\n".join(lines)))
def _clean_parameters(self, parameters):
supported_parameters = self.supported_parameters()
for name, _ in list(parameters.items()):
if name not in supported_parameters:
L.warning(
"Parameter {} not supported for test {}. It will be ignored"
.format(name, self.__class__.__name__))
continue
for name, supported_parameter in list(supported_parameters.items()):
if name not in parameters:
if supported_parameter.required:
raise XVEx("Parameter {} is mandatory for test {}".format(
name, self.__class__.__name__))
else:
L.debug("Defaulting test parameter '{}' to {}".format(
name, copy.deepcopy(supported_parameter.default)))
parameters[name] = supported_parameter.default
return parameters
def _find_windows_pid(self):
# It can take a moment for the PID to register with cygwin. So retry for up to 5 seconds
# if it fails.
timeup = TimeUp(5)
while not timeup:
pids = self._device.pgrep("xv_packet_capture.exe")
L.debug(
"Found the following PIDs matching xv_packet_capture.exe: {}".
format(pids))
for pid in pids:
cmd_line = self._device.command_line_for_pid(pid)
L.debug("Command line for PID {} was: {}".format(
pid, cmd_line))
for arg in cmd_line:
if self._capture_file in arg:
return pid
raise XVEx(
"Couldn't find PID for xv_packet_capture.exe with capture file {}".
format(self._capture_file))