def __init__(self):
self.netTests = []
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffer = None
def __init__(self):
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffers = {}
def setUp(self):
mock_director = MockDirector()
self.measurementManager = MeasurementManager()
self.measurementManager.director = mock_director
self.measurementManager.concurrency = 10
self.measurementManager.retries = 2
self.measurementManager.start()
self.mockNetTest = MockNetTest()
def __init__(self):
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffer = None
def __init__(self):
self.netTests = []
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffer = None
def setUp(self):
mock_director = MockDirector()
self.measurementManager = MeasurementManager()
self.measurementManager.director = mock_director
self.measurementManager.concurrency = 10
self.measurementManager.retries = 2
self.measurementManager.start()
self.mockNetTest = MockNetTest()
def __init__(self):
self.netTests = []
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
def __init__(self):
self.activeNetTests = []
self.activeDecks = []
self.activeMeasurements = {}
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffers = {}
self.input_store = input_store
self.deck_store = deck_store
self._reset_director_state()
self._reset_tor_state()
self._subscribers = []
class TestMeasurementManager(unittest.TestCase):
def setUp(self):
mock_director = MockDirector()
self.measurementManager = MeasurementManager()
self.measurementManager.director = mock_director
self.measurementManager.concurrency = 10
self.measurementManager.retries = 2
self.measurementManager.start()
self.mockNetTest = MockNetTest()
def test_schedule_and_net_test_notified(self, number=1):
# XXX we should probably be inheriting from the base test class
mock_task = MockSuccessMeasurement(self.mockNetTest)
self.measurementManager.schedule(mock_task)
@mock_task.done.addCallback
def done(res):
self.assertEqual(self.mockNetTest.successes,
[42])
self.assertEqual(len(self.mockNetTest.successes), 1)
return mock_task.done
def test_schedule_failing_one_measurement(self):
mock_task = MockFailMeasurement(self.mockNetTest)
self.measurementManager.schedule(mock_task)
@mock_task.done.addErrback
def done(failure):
self.assertEqual(self.measurementManager.failures, 3)
# self.assertEqual(len(self.measurementManager.failures), 3)
self.assertEqual(failure, mockFailure)
self.assertEqual(len(self.mockNetTest.successes), 0)
return mock_task.done
class TestMeasurementManager(unittest.TestCase):
def setUp(self):
mock_director = MockDirector()
self.measurementManager = MeasurementManager()
self.measurementManager.director = mock_director
self.measurementManager.concurrency = 10
self.measurementManager.retries = 2
self.measurementManager.start()
self.mockNetTest = MockNetTest()
def test_schedule_and_net_test_notified(self, number=1):
# XXX we should probably be inheriting from the base test class
mock_task = MockSuccessMeasurement(self.mockNetTest)
self.measurementManager.schedule(mock_task)
@mock_task.done.addCallback
def done(res):
self.assertEqual(self.mockNetTest.successes, [42])
self.assertEqual(len(self.mockNetTest.successes), 1)
return mock_task.done
def test_schedule_failing_one_measurement(self):
mock_task = MockFailMeasurement(self.mockNetTest)
self.measurementManager.schedule(mock_task)
@mock_task.done.addErrback
def done(failure):
self.assertEqual(self.measurementManager.failures, 3)
# self.assertEqual(len(self.measurementManager.failures), 3)
self.assertEqual(failure, mockFailure)
self.assertEqual(len(self.mockNetTest.successes), 0)
return mock_task.done
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager
and the Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
# Only list NetTests belonging to these categories
categories = ['blocking', 'manipulation']
def __init__(self):
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffers = {}
def getNetTests(self):
nettests = {}
def is_nettest(filename):
return not filename == '__init__.py' and filename.endswith('.py')
for category in self.categories:
dirname = os.path.join(config.nettest_directory, category)
# print path to all filenames.
for filename in os.listdir(dirname):
if is_nettest(filename):
net_test_file = os.path.join(dirname, filename)
try:
nettest = getNetTestInformation(net_test_file)
except:
log.err("Error processing %s" % filename)
continue
nettest['category'] = category.replace('/', '')
if nettest['id'] in nettests:
log.err(
"Found a two tests with the same name %s, %s" %
(net_test_file, nettests[nettest['id']]['path']))
else:
category = dirname.replace(config.nettest_directory,
'')
nettests[nettest['id']] = nettest
return nettests
@defer.inlineCallbacks
def start(self, start_tor=False, check_incoherences=True):
self.netTests = self.getNetTests()
if start_tor:
if check_incoherences:
yield config.check_tor()
if config.advanced.start_tor:
yield self.startTor()
elif config.tor.control_port:
log.msg("Connecting to Tor Control Port...")
yield self.getTorState()
if config.global_options['no-geoip']:
aux = [False]
if config.global_options.get('annotations') is not None:
annotations = [
k.lower()
for k in config.global_options['annotations'].keys()
]
aux = map(lambda x: x in annotations,
["city", "country", "asn"])
if not all(aux):
log.msg(
"You should add annotations for the country, city and ASN")
else:
yield config.probe_ip.lookup()
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, result, measurement):
log.debug("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
measurement.result = result
test_name = test_class_name_to_name(measurement.testInstance.name)
if test_name in self.sniffers:
sniffer = self.sniffers[test_name]
config.scapyFactory.unRegisterProtocol(sniffer)
sniffer.close()
del self.sniffers[test_name]
return measurement
def measurementFailed(self, failure, measurement):
log.debug("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
measurement.result = failure
return measurement
def reporterFailed(self, failure, net_test):
"""
This gets called every time a reporter is failing and has been removed
from the reporters of a NetTest.
Once a report has failed to be created that net_test will never use the
reporter again.
XXX hook some logic here.
note: failure contains an extra attribute called failure.reporter
"""
pass
def netTestDone(self, net_test):
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
@defer.inlineCallbacks
def startNetTest(self,
net_test_loader,
report_filename,
collector_address=None):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
"""
if self.allTestsDone.called:
self.allTestsDone = defer.Deferred()
if config.privacy.includepcap:
self.startSniffing(net_test_loader.testDetails)
report = Report(net_test_loader.testDetails, report_filename,
self.reportEntryManager, collector_address)
net_test = NetTest(net_test_loader, report)
net_test.director = self
yield net_test.report.open()
yield net_test.initializeInputProcessor()
try:
self.activeNetTests.append(net_test)
self.measurementManager.schedule(net_test.generateMeasurements())
yield net_test.done
yield report.close()
finally:
self.netTestDone(net_test)
def startSniffing(self, testDetails):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapySniffer, ScapyFactory
if config.scapyFactory is None:
config.scapyFactory = ScapyFactory(config.advanced.interface)
if not config.reports.pcap:
prefix = 'report'
else:
prefix = config.reports.pcap
filename = config.global_options[
'reportfile'] if 'reportfile' in config.global_options.keys(
) else None
filename_pcap = generate_filename(testDetails,
filename=filename,
prefix=prefix,
extension='pcap')
if len(self.sniffers) > 0:
pcap_filenames = set(sniffer.pcapwriter.filename
for sniffer in self.sniffers.values())
pcap_filenames.add(filename_pcap)
log.msg(
"pcap files %s can be messed up because several netTests are being executed in parallel."
% ','.join(pcap_filenames))
sniffer = ScapySniffer(filename_pcap)
self.sniffers[testDetails['test_name']] = sniffer
config.scapyFactory.registerProtocol(sniffer)
log.msg("Starting packet capture to: %s" % filename_pcap)
@defer.inlineCallbacks
def getTorState(self):
connection = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.control_port)
config.tor_state = yield build_tor_connection(connection)
def startTor(self):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
log.msg("Starting Tor...")
@defer.inlineCallbacks
def state_complete(state):
config.tor_state = state
log.msg("Successfully bootstrapped Tor")
log.debug("We now have the following circuits: ")
for circuit in state.circuits.values():
log.debug(" * %s" % circuit)
socks_port = yield state.protocol.get_conf("SocksPort")
control_port = yield state.protocol.get_conf("ControlPort")
config.tor.socks_port = int(socks_port.values()[0])
config.tor.control_port = int(control_port.values()[0])
def setup_failed(failure):
log.exception(failure)
raise errors.UnableToStartTor
def setup_complete(proto):
"""
Called when we read from stdout that Tor has reached 100%.
"""
log.debug("Building a TorState")
config.tor.protocol = proto
state = TorState(proto.tor_protocol)
state.post_bootstrap.addCallback(state_complete)
state.post_bootstrap.addErrback(setup_failed)
return state.post_bootstrap
def updates(prog, tag, summary):
log.msg("%d%%: %s" % (prog, summary))
tor_config = TorConfig()
if config.tor.control_port:
tor_config.ControlPort = config.tor.control_port
if config.tor.socks_port:
tor_config.SocksPort = config.tor.socks_port
if config.tor.data_dir:
data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir):
log.msg("%s does not exist. Creating it." % data_dir)
os.makedirs(data_dir)
tor_config.DataDirectory = data_dir
if config.tor.bridges:
tor_config.UseBridges = 1
if config.advanced.obfsproxy_binary:
tor_config.ClientTransportPlugin = (
'obfs2,obfs3 exec %s managed' %
config.advanced.obfsproxy_binary)
bridges = []
with open(config.tor.bridges) as f:
for bridge in f:
if 'obfs' in bridge:
if config.advanced.obfsproxy_binary:
bridges.append(bridge.strip())
else:
bridges.append(bridge.strip())
tor_config.Bridge = bridges
if config.tor.torrc:
for i in config.tor.torrc.keys():
setattr(tor_config, i, config.tor.torrc[i])
if os.geteuid() == 0:
tor_config.User = pwd.getpwuid(os.geteuid()).pw_name
tor_config.save()
if not hasattr(tor_config, 'ControlPort'):
control_port = int(randomFreePort())
tor_config.ControlPort = control_port
config.tor.control_port = control_port
if not hasattr(tor_config, 'SocksPort'):
socks_port = int(randomFreePort())
tor_config.SocksPort = socks_port
config.tor.socks_port = socks_port
tor_config.save()
log.debug("Setting control port as %s" % tor_config.ControlPort)
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
if config.advanced.tor_binary:
d = launch_tor(tor_config,
reactor,
tor_binary=config.advanced.tor_binary,
progress_updates=updates)
else:
d = launch_tor(tor_config, reactor, progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
return d
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager
and the Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
# Only list NetTests belonging to these categories
categories = ['blocking', 'manipulation', 'third_party']
def __init__(self):
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffers = {}
self.input_store = input_store
self.deck_store = deck_store
self._reset_director_state()
self._reset_tor_state()
self._subscribers = []
def subscribe(self, handler):
self._subscribers.append(handler)
def unsubscribe(self, handler):
self._subscribers.remove(handler)
def notify(self, event):
for handler in self._subscribers:
try:
handler(event)
except Exception as exc:
log.err("Failed to run handler")
log.exception(exc)
def _reset_director_state(self):
self._director_state = 'not-running'
self._director_starting = defer.Deferred()
self._director_starting.addErrback(self._director_startup_failure)
self._director_starting.addCallback(self._director_startup_success)
def _director_startup_failure(self, failure):
self._reset_director_state()
self.notify(DirectorEvent("error", "Failed to start the director"))
return failure
def _director_startup_success(self, result):
self._director_state = 'running'
self.notify(
DirectorEvent("success", "Successfully started the director"))
return result
def _reset_tor_state(self):
# This can be either 'not-running', 'starting' or 'running'
self._tor_state = 'not-running'
self._tor_starting = defer.Deferred()
self._tor_starting.addErrback(self._tor_startup_failure)
self._tor_starting.addCallback(self._tor_startup_success)
def _tor_startup_failure(self, failure):
log.err("Failed to start tor")
log.exception(failure)
self._reset_tor_state()
self.notify(DirectorEvent("error", "Failed to start Tor"))
return failure
def _tor_startup_success(self, result):
log.msg("Tor has started")
self._tor_state = 'running'
self.notify(DirectorEvent("success", "Successfully started Tor"))
return result
def getNetTests(self):
nettests = {}
def is_nettest(filename):
return not filename == '__init__.py' and filename.endswith('.py')
for category in self.categories:
dirname = os.path.join(config.nettest_directory, category)
# print path to all filenames.
for filename in os.listdir(dirname):
if is_nettest(filename):
net_test_file = os.path.join(dirname, filename)
try:
nettest = getNetTestInformation(net_test_file)
except:
log.err("Error processing %s" % filename)
continue
nettest['category'] = category.replace('/', '')
if nettest['id'] in nettests:
log.err(
"Found a two tests with the same name %s, %s" %
(net_test_file, nettests[nettest['id']]['path']))
else:
category = dirname.replace(config.nettest_directory,
'')
nettests[nettest['id']] = nettest
return nettests
@defer.inlineCallbacks
def _start(self, start_tor, check_incoherences, create_input_store):
self.netTests = self.getNetTests()
if start_tor:
yield self.start_tor(check_incoherences)
no_geoip = config.global_options.get('no-geoip', False)
if no_geoip:
aux = [False]
if config.global_options.get('annotations') is not None:
annotations = [
k.lower()
for k in config.global_options['annotations'].keys()
]
aux = map(lambda x: x in annotations,
["city", "country", "asn"])
if not all(aux):
log.msg(
"You should add annotations for the country, city and ASN")
else:
yield probe_ip.lookup()
self.notify(DirectorEvent("success", "Looked up probe IP"))
self.notify(DirectorEvent("success", "Running system tasks"))
yield run_system_tasks(no_input_store=not create_input_store)
self.notify(DirectorEvent("success", "Ran system tasks"))
@defer.inlineCallbacks
def start(self,
start_tor=False,
check_incoherences=True,
create_input_store=True):
self._director_state = 'starting'
try:
yield self._start(start_tor, check_incoherences,
create_input_store)
self._director_starting.callback(self._director_state)
except Exception as exc:
self._director_starting.errback(Failure(exc))
raise
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, result, measurement):
log.debug("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
measurement.result = result
test_name = normalizeTestName(measurement.testInstance.name)
if test_name in self.sniffers:
sniffer = self.sniffers[test_name]
config.scapyFactory.unRegisterProtocol(sniffer)
sniffer.close()
del self.sniffers[test_name]
return measurement
def measurementFailed(self, failure, measurement):
log.debug("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
measurement.result = failure
return measurement
def netTestDone(self, net_test):
self.notify(
DirectorEvent(
"success", "Successfully ran test {0}".format(
net_test.testDetails['test_name'])))
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
@defer.inlineCallbacks
def start_net_test_loader(self,
net_test_loader,
report_filename,
collector_client=None,
no_yamloo=False,
test_details=None,
measurement_id=None):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
"""
if test_details is None:
test_details = net_test_loader.getTestDetails()
test_cases = net_test_loader.getTestCases()
if self.allTestsDone.called:
self.allTestsDone = defer.Deferred()
if config.privacy.includepcap or config.global_options.get(
'pcapfile', None):
self.start_sniffing(test_details)
report = Report(test_details, report_filename, self.reportEntryManager,
collector_client, no_yamloo, measurement_id)
yield report.open()
net_test = NetTest(test_cases, test_details, report)
net_test.director = self
yield net_test.initialize()
try:
self.activeNetTests.append(net_test)
self.measurementManager.schedule(net_test.generateMeasurements())
yield net_test.done
yield report.close()
finally:
self.netTestDone(net_test)
def start_sniffing(self, test_details):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapySniffer, ScapyFactory
if config.scapyFactory is None:
config.scapyFactory = ScapyFactory(config.advanced.interface)
# XXX this is dumb option to have in the ooniprobe.conf. Drop it in
# the future.
prefix = config.reports.pcap
if prefix is None:
prefix = 'report'
filename_pcap = config.global_options.get('pcapfile', None)
if filename_pcap is None:
filename_pcap = generate_filename(test_details,
prefix=prefix,
extension='pcap')
if len(self.sniffers) > 0:
pcap_filenames = set(sniffer.pcapwriter.filename
for sniffer in self.sniffers.values())
pcap_filenames.add(filename_pcap)
log.msg(
"pcap files %s can be messed up because several netTests are being executed in parallel."
% ','.join(pcap_filenames))
sniffer = ScapySniffer(filename_pcap)
self.sniffers[test_details['test_name']] = sniffer
config.scapyFactory.registerProtocol(sniffer)
log.msg("Starting packet capture to: %s" % filename_pcap)
@defer.inlineCallbacks
def start_tor(self, check_incoherences=False):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
if self._tor_state == 'running':
log.debug("Tor is already running")
defer.returnValue(self._tor_state)
elif self._tor_state == 'starting':
log.debug("Tor is starting")
yield self._tor_starting
defer.returnValue(self._tor_state)
log.msg("Starting Tor")
self._tor_state = 'starting'
if check_incoherences:
try:
yield config.check_tor()
except Exception as exc:
self._tor_starting.errback(Failure(exc))
raise exc
if config.advanced.start_tor and config.tor_state is None:
tor_config = get_tor_config()
try:
yield start_tor(tor_config)
self._tor_starting.callback(self._tor_state)
except Exception as exc:
log.err("Failed to start tor")
log.exception(exc)
self._tor_starting.errback(Failure(exc))
elif config.tor.control_port and config.tor_state is None:
try:
yield connect_to_control_port()
self._tor_starting.callback(self._tor_state)
except Exception as exc:
self._tor_starting.errback(Failure(exc))
else:
# This happens when we require tor to not be started and the
# socks port is set.
self._tor_starting.callback(self._tor_state)
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager
and the Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
# Only list NetTests belonging to these categories
categories = ['blocking', 'manipulation']
def __init__(self):
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffers = {}
def getNetTests(self):
nettests = {}
def is_nettest(filename):
return not filename == '__init__.py' and filename.endswith('.py')
for category in self.categories:
dirname = os.path.join(config.nettest_directory, category)
# print path to all filenames.
for filename in os.listdir(dirname):
if is_nettest(filename):
net_test_file = os.path.join(dirname, filename)
try:
nettest = getNetTestInformation(net_test_file)
except:
log.err("Error processing %s" % filename)
continue
nettest['category'] = category.replace('/', '')
if nettest['id'] in nettests:
log.err("Found a two tests with the same name %s, %s" %
(net_test_file,
nettests[nettest['id']]['path']))
else:
category = dirname.replace(config.nettest_directory,
'')
nettests[nettest['id']] = nettest
return nettests
@defer.inlineCallbacks
def start(self, start_tor=False, check_incoherences=True):
self.netTests = self.getNetTests()
if start_tor:
if check_incoherences:
yield config.check_tor()
if config.advanced.start_tor and config.tor_state is None:
yield self.startTor()
elif config.tor.control_port and config.tor_state is None:
log.msg("Connecting to Tor Control Port...")
yield self.getTorState()
if config.global_options['no-geoip']:
aux = [False]
if config.global_options.get('annotations') is not None:
annotations = [k.lower() for k in config.global_options['annotations'].keys()]
aux = map(lambda x: x in annotations, ["city", "country", "asn"])
if not all(aux):
log.msg("You should add annotations for the country, city and ASN")
else:
yield config.probe_ip.lookup()
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, result, measurement):
log.debug("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
measurement.result = result
test_name = test_class_name_to_name(measurement.testInstance.name)
if test_name in self.sniffers:
sniffer = self.sniffers[test_name]
config.scapyFactory.unRegisterProtocol(sniffer)
sniffer.close()
del self.sniffers[test_name]
return measurement
def measurementFailed(self, failure, measurement):
log.debug("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
measurement.result = failure
return measurement
def reporterFailed(self, failure, net_test):
"""
This gets called every time a reporter is failing and has been removed
from the reporters of a NetTest.
Once a report has failed to be created that net_test will never use the
reporter again.
XXX hook some logic here.
note: failure contains an extra attribute called failure.reporter
"""
pass
def netTestDone(self, net_test):
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
@defer.inlineCallbacks
def startNetTest(self, net_test_loader, report_filename,
collector_address=None):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
"""
if self.allTestsDone.called:
self.allTestsDone = defer.Deferred()
if config.privacy.includepcap:
self.startSniffing(net_test_loader.testDetails)
report = Report(net_test_loader.testDetails, report_filename,
self.reportEntryManager, collector_address)
net_test = NetTest(net_test_loader, report)
net_test.director = self
yield net_test.report.open()
yield net_test.initializeInputProcessor()
try:
self.activeNetTests.append(net_test)
self.measurementManager.schedule(net_test.generateMeasurements())
yield net_test.done
yield report.close()
finally:
self.netTestDone(net_test)
def startSniffing(self, testDetails):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapySniffer, ScapyFactory
if config.scapyFactory is None:
config.scapyFactory = ScapyFactory(config.advanced.interface)
if not config.reports.pcap:
prefix = 'report'
else:
prefix = config.reports.pcap
filename = config.global_options['reportfile'] if 'reportfile' in config.global_options.keys() else None
filename_pcap = generate_filename(testDetails, filename=filename, prefix=prefix, extension='pcap')
if len(self.sniffers) > 0:
pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers.values())
pcap_filenames.add(filename_pcap)
log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." %
','.join(pcap_filenames))
sniffer = ScapySniffer(filename_pcap)
self.sniffers[testDetails['test_name']] = sniffer
config.scapyFactory.registerProtocol(sniffer)
log.msg("Starting packet capture to: %s" % filename_pcap)
@defer.inlineCallbacks
def getTorState(self):
connection = TCP4ClientEndpoint(reactor, '127.0.0.1',
config.tor.control_port)
config.tor_state = yield build_tor_connection(connection)
def startTor(self):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
log.msg("Starting Tor...")
@defer.inlineCallbacks
def state_complete(state):
config.tor_state = state
log.msg("Successfully bootstrapped Tor")
log.debug("We now have the following circuits: ")
for circuit in state.circuits.values():
log.debug(" * %s" % circuit)
socks_port = yield state.protocol.get_conf("SocksPort")
control_port = yield state.protocol.get_conf("ControlPort")
config.tor.socks_port = int(socks_port.values()[0])
config.tor.control_port = int(control_port.values()[0])
def setup_failed(failure):
log.exception(failure)
raise errors.UnableToStartTor
def setup_complete(proto):
"""
Called when we read from stdout that Tor has reached 100%.
"""
log.debug("Building a TorState")
config.tor.protocol = proto
state = TorState(proto.tor_protocol)
state.post_bootstrap.addCallback(state_complete)
state.post_bootstrap.addErrback(setup_failed)
return state.post_bootstrap
def updates(prog, tag, summary):
log.msg("%d%%: %s" % (prog, summary))
tor_config = TorConfig()
if config.tor.control_port:
tor_config.ControlPort = config.tor.control_port
if config.tor.socks_port:
tor_config.SocksPort = config.tor.socks_port
if config.tor.data_dir:
data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir):
log.msg("%s does not exist. Creating it." % data_dir)
os.makedirs(data_dir)
tor_config.DataDirectory = data_dir
if config.tor.bridges:
tor_config.UseBridges = 1
if config.advanced.obfsproxy_binary:
tor_config.ClientTransportPlugin = (
'obfs2,obfs3 exec %s managed' %
config.advanced.obfsproxy_binary
)
bridges = []
with open(config.tor.bridges) as f:
for bridge in f:
if 'obfs' in bridge:
if config.advanced.obfsproxy_binary:
bridges.append(bridge.strip())
else:
bridges.append(bridge.strip())
tor_config.Bridge = bridges
if config.tor.torrc:
for i in config.tor.torrc.keys():
setattr(tor_config, i, config.tor.torrc[i])
if os.geteuid() == 0:
tor_config.User = pwd.getpwuid(os.geteuid()).pw_name
tor_config.save()
if not hasattr(tor_config, 'ControlPort'):
control_port = int(randomFreePort())
tor_config.ControlPort = control_port
config.tor.control_port = control_port
if not hasattr(tor_config, 'SocksPort'):
socks_port = int(randomFreePort())
tor_config.SocksPort = socks_port
config.tor.socks_port = socks_port
tor_config.save()
log.debug("Setting control port as %s" % tor_config.ControlPort)
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
if config.advanced.tor_binary:
d = launch_tor(tor_config, reactor,
tor_binary=config.advanced.tor_binary,
progress_updates=updates)
else:
d = launch_tor(tor_config, reactor,
progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
return d
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager and the
Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
def __init__(self):
self.netTests = []
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffer = None
@defer.inlineCallbacks
def start(self):
if config.advanced.start_tor:
log.msg("Starting Tor...")
yield self.startTor()
config.probe_ip = geoip.ProbeIP()
yield config.probe_ip.lookup()
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, measurement):
log.msg("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
return measurement.testInstance.report
def measurementFailed(self, failure, measurement):
log.msg("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
self.failures.append((failure, measurement))
return None
def reporterFailed(self, failure, net_test):
"""
This gets called every time a reporter is failing and has been removed
from the reporters of a NetTest.
Once a report has failed to be created that net_test will never use the
reporter again.
XXX hook some logic here.
note: failure contains an extra attribute called failure.reporter
"""
pass
def netTestDone(self, result, net_test):
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
self.allTestsDone = defer.Deferred()
@defer.inlineCallbacks
def startNetTest(self, _, net_test_loader, reporters):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
_: #XXX very dirty hack
"""
if config.privacy.includepcap:
log.msg("Starting")
if not config.reports.pcap:
config.reports.pcap = config.generatePcapFilename(net_test_loader.testDetails)
self.startSniffing()
report = Report(reporters, self.reportEntryManager)
net_test = NetTest(net_test_loader, report)
net_test.director = self
yield net_test.report.open()
self.measurementManager.schedule(net_test.generateMeasurements())
self.activeNetTests.append(net_test)
net_test.done.addBoth(report.close)
net_test.done.addBoth(self.netTestDone, net_test)
yield net_test.done
def startSniffing(self):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapyFactory, ScapySniffer
try:
checkForRoot()
except errors.InsufficientPrivileges:
print "[!] Includepcap options requires root priviledges to run"
print " you should run ooniprobe as root or disable the options in ooniprobe.conf"
reactor.stop()
sys.exit(1)
print "Starting sniffer"
config.scapyFactory = ScapyFactory(config.advanced.interface)
if os.path.exists(config.reports.pcap):
print "Report PCAP already exists with filename %s" % config.reports.pcap
print "Renaming files with such name..."
pushFilenameStack(config.reports.pcap)
if self.sniffer:
config.scapyFactory.unRegisterProtocol(self.sniffer)
self.sniffer = ScapySniffer(config.reports.pcap)
config.scapyFactory.registerProtocol(self.sniffer)
def startTor(self):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
@defer.inlineCallbacks
def state_complete(state):
config.tor_state = state
log.msg("Successfully bootstrapped Tor")
log.debug("We now have the following circuits: ")
for circuit in state.circuits.values():
log.debug(" * %s" % circuit)
socks_port = yield state.protocol.get_conf("SocksPort")
control_port = yield state.protocol.get_conf("ControlPort")
config.tor.socks_port = int(socks_port.values()[0])
config.tor.control_port = int(control_port.values()[0])
log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip)
def setup_failed(failure):
log.exception(failure)
raise errors.UnableToStartTor
def setup_complete(proto):
"""
Called when we read from stdout that Tor has reached 100%.
"""
log.debug("Building a TorState")
state = TorState(proto.tor_protocol)
state.post_bootstrap.addCallback(state_complete)
state.post_bootstrap.addErrback(setup_failed)
return state.post_bootstrap
def updates(prog, tag, summary):
log.debug("%d%%: %s" % (prog, summary))
tor_config = TorConfig()
if config.tor.control_port:
tor_config.ControlPort = config.tor.control_port
else:
control_port = int(randomFreePort())
tor_config.ControlPort = control_port
config.tor.control_port = control_port
if config.tor.socks_port:
tor_config.SocksPort = config.tor.socks_port
else:
socks_port = int(randomFreePort())
tor_config.SocksPort = socks_port
config.tor.socks_port = socks_port
if config.tor.data_dir:
data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir):
log.msg("%s does not exist. Creating it." % data_dir)
os.makedirs(data_dir)
tor_config.DataDirectory = data_dir
tor_config.save()
log.debug("Setting control port as %s" % tor_config.ControlPort)
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
if config.advanced.tor_binary:
d = launch_tor(tor_config, reactor,
tor_binary=config.advanced.tor_binary,
progress_updates=updates)
else:
d = launch_tor(tor_config, reactor,
progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
return d
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager
and the Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
# Only list NetTests belonging to these categories
categories = ['blocking', 'manipulation', 'third_party']
def __init__(self):
self.activeNetTests = []
self.activeDecks = []
self.activeMeasurements = {}
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffers = {}
self.input_store = input_store
self.deck_store = deck_store
self._reset_director_state()
self._reset_tor_state()
self._subscribers = []
def subscribe(self, handler):
self._subscribers.append(handler)
def unsubscribe(self, handler):
self._subscribers.remove(handler)
def notify(self, event):
for handler in self._subscribers:
try:
handler(event)
except Exception as exc:
log.err("Failed to run handler")
log.exception(exc)
def _reset_director_state(self):
self._director_state = 'not-running'
self._director_starting = defer.Deferred()
self._director_starting.addErrback(self._director_startup_failure)
self._director_starting.addCallback(self._director_startup_success)
def _director_startup_failure(self, failure):
self._reset_director_state()
self.notify(DirectorEvent("error",
"Failed to start the director"))
return failure
def _director_startup_success(self, result):
self._director_state = 'running'
self.notify(DirectorEvent("success",
"Successfully started the director"))
return result
def _reset_tor_state(self):
# This can be either 'not-running', 'starting' or 'running'
self._tor_state = 'not-running'
self._tor_starting = defer.Deferred()
self._tor_starting.addErrback(self._tor_startup_failure)
self._tor_starting.addCallback(self._tor_startup_success)
def _tor_startup_failure(self, failure):
log.err("Failed to start tor")
log.exception(failure)
self._reset_tor_state()
self.notify(DirectorEvent("error",
"Failed to start Tor"))
return failure
def _tor_startup_success(self, result):
log.msg("Tor has started")
self._tor_state = 'running'
self.notify(DirectorEvent("success",
"Successfully started Tor"))
return result
def getNetTests(self):
nettests = {}
def is_nettest(filename):
return not filename == '__init__.py' and filename.endswith('.py')
for category in self.categories:
dirname = os.path.join(config.nettest_directory, category)
# print path to all filenames.
for filename in os.listdir(dirname):
if is_nettest(filename):
net_test_file = os.path.join(dirname, filename)
try:
nettest = getNetTestInformation(net_test_file)
except:
log.err("Error processing %s" % filename)
continue
nettest['category'] = category.replace('/', '')
if nettest['id'] in nettests:
log.err("Found a two tests with the same name %s, %s" %
(net_test_file,
nettests[nettest['id']]['path']))
else:
category = dirname.replace(config.nettest_directory,
'')
nettests[nettest['id']] = nettest
return nettests
@defer.inlineCallbacks
def _start(self, start_tor, check_incoherences, create_input_store):
self.netTests = self.getNetTests()
if start_tor:
yield self.start_tor(check_incoherences)
no_geoip = config.global_options.get('no-geoip', False)
if no_geoip:
aux = [False]
if config.global_options.get('annotations') is not None:
annotations = [k.lower() for k in config.global_options['annotations'].keys()]
aux = map(lambda x: x in annotations, ["city", "country", "asn"])
if not all(aux):
log.msg("You should add annotations for the country, city and ASN")
else:
yield probe_ip.lookup()
self.notify(DirectorEvent("success", "Looked up probe IP"))
self.notify(DirectorEvent("success", "Running system tasks"))
yield run_system_tasks(no_input_store=not create_input_store)
self.notify(DirectorEvent("success", "Ran system tasks"))
@defer.inlineCallbacks
def start(self, start_tor=False, check_incoherences=True,
create_input_store=True):
self._director_state = 'starting'
try:
yield self._start(start_tor, check_incoherences, create_input_store)
self._director_starting.callback(self._director_state)
except Exception as exc:
self._director_starting.errback(Failure(exc))
raise
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, result, measurement):
log.debug("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
measurement.result = result
test_name = normalizeTestName(measurement.testInstance.name)
if test_name in self.sniffers:
sniffer = self.sniffers[test_name]
config.scapyFactory.unRegisterProtocol(sniffer)
sniffer.close()
del self.sniffers[test_name]
return measurement
def measurementFailed(self, failure, measurement):
log.debug("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
measurement.result = failure
return measurement
def deckStarted(self, deck_id, from_schedule):
log.debug("Starting {0} ({1})".format(deck_id,
'scheduled' if from_schedule
else 'user-run'))
self.activeDecks.append((deck_id, from_schedule))
def deckFinished(self, deck_id, from_schedule):
log.debug("Finished {0} ({1})".format(deck_id,
'scheduled' if from_schedule
else 'user-run'))
try:
self.activeDecks.remove((deck_id, from_schedule))
except ValueError:
log.error("Completed deck {0} is not actually running".format(
deck_id))
def isDeckRunning(self, deck_id, from_schedule):
"""
:param deck_id: the ID of the deck to check if it's running
:param from_schedule: True if we want to know the status of a
scheduled deck run False for user initiated runs.
:return: True if the deck is running False otherwise
"""
return (deck_id, from_schedule) in self.activeDecks
def netTestDone(self, net_test):
self.notify(DirectorEvent("success",
"Successfully ran test {0}".format(
net_test.testDetails['test_name'])))
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
@defer.inlineCallbacks
def start_net_test_loader(self, net_test_loader, report_filename,
collector_client=None, no_yamloo=False,
test_details=None, measurement_id=None):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
"""
if test_details is None:
test_details = net_test_loader.getTestDetails()
test_cases = net_test_loader.getTestCases()
if self.allTestsDone.called:
self.allTestsDone = defer.Deferred()
if config.privacy.includepcap or config.global_options.get('pcapfile', None):
self.start_sniffing(test_details)
report = Report(test_details, report_filename,
self.reportEntryManager,
collector_client,
no_yamloo,
measurement_id)
yield report.open()
net_test = NetTest(test_cases, test_details, report)
net_test.director = self
yield net_test.initialize()
try:
self.activeNetTests.append(net_test)
if measurement_id:
self.activeMeasurements[measurement_id] = net_test
self.measurementManager.schedule(net_test.generateMeasurements())
yield net_test.done
yield report.close()
finally:
if measurement_id:
del self.activeMeasurements[measurement_id]
self.netTestDone(net_test)
def start_sniffing(self, test_details):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapySniffer, ScapyFactory
if config.scapyFactory is None:
config.scapyFactory = ScapyFactory(config.advanced.interface)
# XXX this is dumb option to have in the ooniprobe.conf. Drop it in
# the future.
prefix = config.reports.pcap
if prefix is None:
prefix = 'report'
filename_pcap = config.global_options.get('pcapfile', None)
if filename_pcap is None:
filename_pcap = generate_filename(test_details,
prefix=prefix,
extension='pcap')
if len(self.sniffers) > 0:
pcap_filenames = set(sniffer.pcapwriter.filename for sniffer in self.sniffers.values())
pcap_filenames.add(filename_pcap)
log.msg("pcap files %s can be messed up because several netTests are being executed in parallel." %
','.join(pcap_filenames))
sniffer = ScapySniffer(filename_pcap)
self.sniffers[test_details['test_name']] = sniffer
config.scapyFactory.registerProtocol(sniffer)
log.msg("Starting packet capture to: %s" % filename_pcap)
@defer.inlineCallbacks
def start_tor(self, check_incoherences=False):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
if self._tor_state == 'running':
log.debug("Tor is already running")
defer.returnValue(self._tor_state)
elif self._tor_state == 'starting':
log.debug("Tor is starting")
yield self._tor_starting
defer.returnValue(self._tor_state)
log.msg("Starting Tor")
self._tor_state = 'starting'
if check_incoherences:
try:
yield config.check_tor()
except Exception as exc:
self._tor_starting.errback(Failure(exc))
raise exc
if config.advanced.start_tor and config.tor_state is None:
tor_config = get_tor_config()
try:
yield start_tor(tor_config)
self._tor_starting.callback(self._tor_state)
except Exception as exc:
log.err("Failed to start tor")
log.exception(exc)
self._tor_starting.errback(Failure(exc))
elif config.tor.control_port and config.tor_state is None:
try:
yield connect_to_control_port()
self._tor_starting.callback(self._tor_state)
except Exception as exc:
self._tor_starting.errback(Failure(exc))
else:
# This happens when we require tor to not be started and the
# socks port is set.
self._tor_starting.callback(self._tor_state)
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager and the
Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
# Only list NetTests belonging to these categories
categories = ['blocking', 'manipulation']
def __init__(self):
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
# Link the TaskManager's by least available slots.
self.measurementManager.child = self.reportEntryManager
# Notify the parent when tasks complete # XXX deadlock!?
self.reportEntryManager.parent = self.measurementManager
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffer = None
def getNetTests(self):
nettests = {}
def is_nettest(filename):
return not filename == '__init__.py' \
and filename.endswith('.py')
for category in self.categories:
dirname = os.path.join(config.nettest_directory, category)
# print path to all filenames.
for filename in os.listdir(dirname):
if is_nettest(filename):
net_test_file = os.path.join(dirname, filename)
nettest = getNetTestInformation(net_test_file)
if nettest['id'] in nettests:
log.err("Found a two tests with the same name %s, %s" %
(nettest_path, nettests[nettest['id']]['path']))
else:
category = dirname.replace(config.nettest_directory, '')
nettests[nettest['id']] = nettest
return nettests
@defer.inlineCallbacks
def start(self):
self.netTests = self.getNetTests()
if config.advanced.start_tor:
log.msg("Starting Tor...")
yield self.startTor()
config.probe_ip = geoip.ProbeIP()
yield config.probe_ip.lookup()
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, measurement):
log.msg("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
return measurement
def measurementFailed(self, failure, measurement):
log.msg("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
self.failures.append((failure, measurement))
return measurement
def reporterFailed(self, failure, net_test):
"""
This gets called every time a reporter is failing and has been removed
from the reporters of a NetTest.
Once a report has failed to be created that net_test will never use the
reporter again.
XXX hook some logic here.
note: failure contains an extra attribute called failure.reporter
"""
pass
def netTestDone(self, net_test):
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
self.allTestsDone = defer.Deferred()
@defer.inlineCallbacks
def startNetTest(self, net_test_loader, reporters):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
"""
if config.privacy.includepcap:
if not config.reports.pcap:
config.reports.pcap = config.generatePcapFilename(net_test_loader.testDetails)
self.startSniffing()
report = Report(reporters, self.reportEntryManager)
net_test = NetTest(net_test_loader, report)
net_test.director = self
yield net_test.report.open()
yield net_test.initializeInputProcessor()
self.measurementManager.schedule(net_test.generateMeasurements())
self.activeNetTests.append(net_test)
yield net_test.done
yield report.close()
self.netTestDone(net_test)
def startSniffing(self):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapyFactory, ScapySniffer
config.scapyFactory = ScapyFactory(config.advanced.interface)
if os.path.exists(config.reports.pcap):
log.msg("Report PCAP already exists with filename %s" % config.reports.pcap)
log.msg("Renaming files with such name...")
pushFilenameStack(config.reports.pcap)
if self.sniffer:
config.scapyFactory.unRegisterProtocol(self.sniffer)
self.sniffer = ScapySniffer(config.reports.pcap)
config.scapyFactory.registerProtocol(self.sniffer)
log.msg("Starting packet capture to: %s" % config.reports.pcap)
def startTor(self):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
@defer.inlineCallbacks
def state_complete(state):
config.tor_state = state
log.msg("Successfully bootstrapped Tor")
log.debug("We now have the following circuits: ")
for circuit in state.circuits.values():
log.debug(" * %s" % circuit)
socks_port = yield state.protocol.get_conf("SocksPort")
control_port = yield state.protocol.get_conf("ControlPort")
config.tor.socks_port = int(socks_port.values()[0])
config.tor.control_port = int(control_port.values()[0])
log.debug("Obtained our IP address from a Tor Relay %s" % config.probe_ip)
def setup_failed(failure):
log.exception(failure)
raise errors.UnableToStartTor
def setup_complete(proto):
"""
Called when we read from stdout that Tor has reached 100%.
"""
log.debug("Building a TorState")
state = TorState(proto.tor_protocol)
state.post_bootstrap.addCallback(state_complete)
state.post_bootstrap.addErrback(setup_failed)
return state.post_bootstrap
def updates(prog, tag, summary):
log.debug("%d%%: %s" % (prog, summary))
tor_config = TorConfig()
if config.tor.control_port:
tor_config.ControlPort = config.tor.control_port
else:
control_port = int(randomFreePort())
tor_config.ControlPort = control_port
config.tor.control_port = control_port
if config.tor.socks_port:
tor_config.SocksPort = config.tor.socks_port
else:
socks_port = int(randomFreePort())
tor_config.SocksPort = socks_port
config.tor.socks_port = socks_port
if config.tor.data_dir:
data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir):
log.msg("%s does not exist. Creating it." % data_dir)
os.makedirs(data_dir)
tor_config.DataDirectory = data_dir
if config.tor.bridges:
tor_config.UseBridges = 1
if config.advanced.obfsproxy_binary:
tor_config.ClientTransportPlugin = \
'obfs2,obfs3 exec %s managed' % \
config.advanced.obfsproxy_binary
bridges = []
with open(config.tor.bridges) as f:
for bridge in f:
if 'obfs' in bridge:
if config.advanced.obfsproxy_binary:
bridges.append(bridge.strip())
else:
bridges.append(bridge.strip())
tor_config.Bridge = bridges
tor_config.save()
log.debug("Setting control port as %s" % tor_config.ControlPort)
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
if config.advanced.tor_binary:
d = launch_tor(tor_config, reactor,
tor_binary=config.advanced.tor_binary,
progress_updates=updates)
else:
d = launch_tor(tor_config, reactor,
progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
return d
class Director(object):
"""
Singleton object responsible for coordinating the Measurements Manager and the
Reporting Manager.
How this all looks like is as follows:
+------------------------------------------------+
| Director |<--+
+------------------------------------------------+ |
^ ^ |
| Measurement | |
+---------+ [---------] +--------------------+ |
| | | MeasurementManager | |
| NetTest | [---------] +--------------------+ |
| | | [----------------] | |
+---------+ [---------] | [----------------] | |
| | [----------------] | |
| +--------------------+ |
v |
+---------+ ReportEntry |
| | [---------] +--------------------+ |
| Report | | ReportEntryManager | |
| | [---------] +--------------------+ |
+---------+ | [----------------] | |
[---------] | [----------------] |--
| [----------------] |
+--------------------+
[------------] are Tasks
+------+
| | are TaskManagers
+------+
| |
+------+
+------+
| | are general purpose objects
+------+
"""
_scheduledTests = 0
def __init__(self):
self.netTests = []
self.activeNetTests = []
self.measurementManager = MeasurementManager()
self.measurementManager.director = self
self.reportEntryManager = ReportEntryManager()
self.reportEntryManager.director = self
self.successfulMeasurements = 0
self.failedMeasurements = 0
self.totalMeasurements = 0
# The cumulative runtime of all the measurements
self.totalMeasurementRuntime = 0
self.failures = []
self.torControlProtocol = None
# This deferred is fired once all the measurements and their reporting
# tasks are completed.
self.allTestsDone = defer.Deferred()
self.sniffer = None
@defer.inlineCallbacks
def start(self):
if config.advanced.start_tor:
log.msg("Starting Tor...")
yield self.startTor()
config.probe_ip = geoip.ProbeIP()
yield config.probe_ip.lookup()
@property
def measurementSuccessRatio(self):
if self.totalMeasurements == 0:
return 0
return self.successfulMeasurements / self.totalMeasurements
@property
def measurementFailureRatio(self):
if self.totalMeasurements == 0:
return 0
return self.failedMeasurements / self.totalMeasurements
@property
def measurementSuccessRate(self):
"""
The speed at which tests are succeeding globally.
This means that fast tests that perform a lot of measurements will
impact this value quite heavily.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.successfulMeasurements / self.totalMeasurementRuntime
@property
def measurementFailureRate(self):
"""
The speed at which tests are failing globally.
"""
if self.totalMeasurementRuntime == 0:
return 0
return self.failedMeasurements / self.totalMeasurementRuntime
def measurementTimedOut(self, measurement):
"""
This gets called every time a measurement times out independenty from
the fact that it gets re-scheduled or not.
"""
pass
def measurementStarted(self, measurement):
self.totalMeasurements += 1
def measurementSucceeded(self, measurement):
log.msg("Successfully completed measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.successfulMeasurements += 1
return measurement.testInstance.report
def measurementFailed(self, failure, measurement):
log.msg("Failed doing measurement: %s" % measurement)
self.totalMeasurementRuntime += measurement.runtime
self.failedMeasurements += 1
self.failures.append((failure, measurement))
return None
def reporterFailed(self, failure, net_test):
"""
This gets called every time a reporter is failing and has been removed
from the reporters of a NetTest.
Once a report has failed to be created that net_test will never use the
reporter again.
XXX hook some logic here.
note: failure contains an extra attribute called failure.reporter
"""
pass
def netTestDone(self, result, net_test):
self.activeNetTests.remove(net_test)
if len(self.activeNetTests) == 0:
self.allTestsDone.callback(None)
self.allTestsDone = defer.Deferred()
@defer.inlineCallbacks
def startNetTest(self, _, net_test_loader, reporters):
"""
Create the Report for the NetTest and start the report NetTest.
Args:
net_test_loader:
an instance of :class:ooni.nettest.NetTestLoader
_: #XXX very dirty hack
"""
if config.privacy.includepcap:
log.msg("Starting")
if not config.reports.pcap:
config.reports.pcap = config.generatePcapFilename(
net_test_loader.testDetails)
self.startSniffing()
report = Report(reporters, self.reportEntryManager)
net_test = NetTest(net_test_loader, report)
net_test.director = self
yield net_test.report.open()
self.measurementManager.schedule(net_test.generateMeasurements())
self.activeNetTests.append(net_test)
net_test.done.addBoth(report.close)
net_test.done.addBoth(self.netTestDone, net_test)
yield net_test.done
def startSniffing(self):
""" Start sniffing with Scapy. Exits if required privileges (root) are not
available.
"""
from ooni.utils.txscapy import ScapyFactory, ScapySniffer
try:
checkForRoot()
except errors.InsufficientPrivileges:
print "[!] Includepcap options requires root priviledges to run"
print " you should run ooniprobe as root or disable the options in ooniprobe.conf"
reactor.stop()
sys.exit(1)
print "Starting sniffer"
config.scapyFactory = ScapyFactory(config.advanced.interface)
if os.path.exists(config.reports.pcap):
print "Report PCAP already exists with filename %s" % config.reports.pcap
print "Renaming files with such name..."
pushFilenameStack(config.reports.pcap)
if self.sniffer:
config.scapyFactory.unRegisterProtocol(self.sniffer)
self.sniffer = ScapySniffer(config.reports.pcap)
config.scapyFactory.registerProtocol(self.sniffer)
def startTor(self):
""" Starts Tor
Launches a Tor with :param: socks_port :param: control_port
:param: tor_binary set in ooniprobe.conf
"""
@defer.inlineCallbacks
def state_complete(state):
config.tor_state = state
log.msg("Successfully bootstrapped Tor")
log.debug("We now have the following circuits: ")
for circuit in state.circuits.values():
log.debug(" * %s" % circuit)
socks_port = yield state.protocol.get_conf("SocksPort")
control_port = yield state.protocol.get_conf("ControlPort")
config.tor.socks_port = int(socks_port.values()[0])
config.tor.control_port = int(control_port.values()[0])
log.debug("Obtained our IP address from a Tor Relay %s" %
config.probe_ip)
def setup_failed(failure):
log.exception(failure)
raise errors.UnableToStartTor
def setup_complete(proto):
"""
Called when we read from stdout that Tor has reached 100%.
"""
log.debug("Building a TorState")
state = TorState(proto.tor_protocol)
state.post_bootstrap.addCallback(state_complete)
state.post_bootstrap.addErrback(setup_failed)
return state.post_bootstrap
def updates(prog, tag, summary):
log.debug("%d%%: %s" % (prog, summary))
tor_config = TorConfig()
if config.tor.control_port:
tor_config.ControlPort = config.tor.control_port
else:
control_port = int(randomFreePort())
tor_config.ControlPort = control_port
config.tor.control_port = control_port
if config.tor.socks_port:
tor_config.SocksPort = config.tor.socks_port
else:
socks_port = int(randomFreePort())
tor_config.SocksPort = socks_port
config.tor.socks_port = socks_port
if config.tor.data_dir:
data_dir = os.path.expanduser(config.tor.data_dir)
if not os.path.exists(data_dir):
log.msg("%s does not exist. Creating it." % data_dir)
os.makedirs(data_dir)
tor_config.DataDirectory = data_dir
tor_config.save()
log.debug("Setting control port as %s" % tor_config.ControlPort)
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
if config.advanced.tor_binary:
d = launch_tor(tor_config,
reactor,
tor_binary=config.advanced.tor_binary,
progress_updates=updates)
else:
d = launch_tor(tor_config, reactor, progress_updates=updates)
d.addCallback(setup_complete)
d.addErrback(setup_failed)
return d
