def __init__(self):
self.current_user = getpass.getuser()
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = geoip.ProbeIP()
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
def __init__(self):
self.current_user = getpass.getuser()
self.global_options = {}
self.scapyFactory = None
self.tor_state = None
self.logging = True
# These are the configuration options
self.basic = Storage()
self.advanced = Storage()
self.reports = Storage()
self.tor = Storage()
self.privacy = Storage()
# In here we store the configuration files ordered by priority.
# First configuration file takes priority over the others.
self.config_files = []
self.set_paths()
def __init__(self):
self.current_user = getpass.getuser()
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = geoip.ProbeIP()
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
def __init__(self):
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = None
# This is used to keep track of the state of the sniffer
self.sniffer_running = None
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
self.initialize_ooni_home()
class OConfig(object):
def __init__(self):
self.current_user = getpass.getuser()
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = None
# This is used to keep track of the state of the sniffer
self.sniffer_running = None
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
self.initialize_ooni_home()
def set_paths(self):
if self.global_options.get("datadir"):
self.data_directory = abspath(expanduser(self.global_options["datadir"]))
elif self.advanced.get("data_dir"):
self.data_directory = self.advanced["data_dir"]
elif hasattr(sys, "real_prefix"):
self.data_directory = os.path.abspath(os.path.join(sys.prefix, "share", "ooni"))
else:
self.data_directory = "/usr/share/ooni/"
self.nettest_directory = abspath(os.path.join(__file__, "..", "nettests"))
self.ooni_home = os.path.join(expanduser("~" + self.current_user), ".ooni")
self.inputs_directory = os.path.join(self.ooni_home, "inputs")
self.decks_directory = os.path.join(self.ooni_home, "decks")
self.reports_directory = os.path.join(self.ooni_home, "reports")
if self.global_options.get("configfile"):
config_file = self.global_options["configfile"]
else:
config_file = os.path.join("~", ".ooni", "ooniprobe.conf")
self.config_file = expanduser(config_file)
def initialize_ooni_home(self):
if not os.path.isdir(self.ooni_home):
print "Ooni home directory does not exist."
print "Creating it in '%s'." % self.ooni_home
os.mkdir(self.ooni_home)
os.mkdir(self.inputs_directory)
os.mkdir(self.decks_directory)
if not os.path.isdir(self.reports_directory):
os.mkdir(self.reports_directory)
def _create_config_file(self):
sample_config_file = os.path.join(self.data_directory, "ooniprobe.conf.sample")
target_config_file = os.path.join(self.ooni_home, "ooniprobe.conf")
print "Creating it for you in '%s'." % target_config_file
usr_share_path = "/usr/share"
if hasattr(sys, "real_prefix"):
usr_share_path = os.path.abspath(os.path.join(sys.prefix, "share"))
with open(sample_config_file) as f:
with open(target_config_file, "w+") as w:
for line in f:
if line.startswith(" data_dir: "):
w.write(" data_dir: %s\n" % os.path.join(usr_share_path, "ooni"))
elif line.startswith(" geoip_data_dir: "):
w.write(" geoip_data_dir: %s\n" % os.path.join(usr_share_path, "GeoIP"))
else:
w.write(line)
def read_config_file(self):
try:
with open(self.config_file) as f:
pass
except IOError:
print "Configuration file does not exist."
self._create_config_file()
self.read_config_file()
with open(self.config_file) as f:
config_file_contents = "\n".join(f.readlines())
configuration = yaml.safe_load(config_file_contents)
for setting in ["basic", "reports", "advanced", "privacy", "tor"]:
try:
for k, v in configuration[setting].items():
getattr(self, setting)[k] = v
except AttributeError:
pass
self.set_paths()
def generatePcapFilename(self, testDetails):
test_name, start_time = testDetails["test_name"], testDetails["start_time"]
start_time = otime.epochToTimestamp(start_time)
return "report-%s-%s.%s" % (test_name, start_time, "pcap")
class OConfig(object):
def __init__(self):
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = None
# This is used to keep track of the state of the sniffer
self.sniffer_running = None
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
self.initialize_ooni_home()
def set_paths(self):
if self.global_options.get('datadir'):
self.data_directory = abspath(expanduser(self.global_options['datadir']))
elif self.advanced.get('data_dir'):
self.data_directory = self.advanced['data_dir']
else:
self.data_directory = '/usr/share/ooni/'
self.nettest_directory = abspath(os.path.join(__file__, '..', 'nettests'))
self.ooni_home = os.path.join(expanduser('~'), '.ooni')
self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
self.reports_directory = os.path.join(self.ooni_home, 'reports')
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
else:
config_file = os.path.join('~', '.ooni', 'ooniprobe.conf')
self.config_file = expanduser(config_file)
def initialize_ooni_home(self):
if not os.path.isdir(self.ooni_home):
print "Ooni home directory does not exist."
print "Creating it in '%s'." % self.ooni_home
os.mkdir(self.ooni_home)
os.mkdir(self.inputs_directory)
if not os.path.isdir(self.reports_directory):
os.mkdir(self.reports_directory)
def _create_config_file(self):
sample_config_file = os.path.join(self.data_directory,
'ooniprobe.conf.sample')
target_config_file = os.path.join(self.ooni_home,
'ooniprobe.conf')
print "Creating it for you in '%s'." % target_config_file
copyfile(sample_config_file, target_config_file)
def read_config_file(self):
try:
with open(self.config_file) as f: pass
except IOError:
print "Configuration file does not exist."
self._create_config_file()
self.read_config_file()
with open(self.config_file) as f:
config_file_contents = '\n'.join(f.readlines())
configuration = yaml.safe_load(config_file_contents)
for setting in ['basic', 'advanced', 'privacy', 'tor']:
try:
for k, v in configuration[setting].items():
getattr(self, setting)[k] = v
except AttributeError:
pass
self.set_paths()
def generatePcapFilename(self, testDetails):
test_name, start_time = testDetails['test_name'], testDetails['start_time']
start_time = otime.epochToTimestamp(start_time)
return "report-%s-%s.%s" % (test_name, start_time, "pcap")
from ooni.utils import Storage
import os
def get_root_path():
this_directory = os.path.dirname(__file__)
root = os.path.join(this_directory, '..')
root = os.path.abspath(root)
return root
# XXX convert this to something that is a proper config file
main = Storage()
# This is the location where submitted reports get stored
main.report_dir = os.path.join(get_root_path(), 'oonib', 'reports')
# This is where tor will place it's Hidden Service hostname and Hidden service
# private key
main.tor_datadir = os.path.join(get_root_path(), 'oonib', 'data', 'tor')
main.database_uri = "sqlite:" + get_root_path() + "oonib_test_db.db"
main.db_threadpool_size = 10
helpers = Storage()
helpers.http_return_request = Storage()
helpers.http_return_request.port = 57001
# XXX this actually needs to be the advertised Server HTTP header of our web
# server
helpers.http_return_request.server_version = "Apache"
from ooni.utils import Storage
import os
def get_root_path():
this_directory = os.path.dirname(__file__)
root = os.path.join(this_directory, '..')
root = os.path.abspath(root)
return root
# XXX convert this to something that is a proper config file
main = Storage()
# This is the location where submitted reports get stored
main.report_dir = os.path.join(get_root_path(), 'oonib', 'reports')
# This is where tor will place it's Hidden Service hostname and Hidden service
# private key
main.tor_datadir = os.path.join(get_root_path(), 'oonib', 'data', 'tor')
main.database_uri = "sqlite:"+get_root_path()+"oonib_test_db.db"
main.db_threadpool_size = 10
helpers = Storage()
helpers.http_return_request = Storage()
helpers.http_return_request.port = 57001
# XXX this actually needs to be the advertised Server HTTP header of our web
# server
helpers.http_return_request.server_version = "Apache"
helpers.tcp_echo = Storage()
from ooni.utils import Storage
from ooni.utils.config import Config
import os
root = os.path.normpath(os.path.join(os.path.realpath(__file__), '../../'))
config = Storage()
config.main = Config('main', os.path.join(root, 'oonibackend.conf'))
config.daphn3 = Config('daphn3', os.path.join(root, 'oonibackend.conf'))
from ooni.utils import Storage # XXX move this to an actual configuration file basic = Storage() advanced = Storage() advanced.debug = True
class OConfig(object):
_custom_home = None
def __init__(self):
self.current_user = getpass.getuser()
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = geoip.ProbeIP()
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
@property
def data_directory(self):
data_directory = abspath(os.path.join(__file__, '..', '..', 'data'))
if os.getenv("OONI_DATA_DIR"):
data_directory = os.getenv("OONI_DATA_DIR")
elif self.global_options.get('datadir'):
data_directory = abspath(expanduser(self.global_options['datadir']))
elif self.advanced.get('data_dir'):
data_directory = self.advanced['data_dir']
elif hasattr(sys, 'real_prefix'):
data_directory = os.path.abspath(os.path.join(sys.prefix, 'share', 'ooni'))
elif not os.path.exists(data_directory):
data_directory = '/usr/share/ooni/'
return data_directory
def set_paths(self, ooni_home=None):
if ooni_home:
self._custom_home = ooni_home
self.nettest_directory = abspath(os.path.join(__file__, '..', 'nettests'))
self.ooni_home = os.path.join(expanduser('~'+self.current_user), '.ooni')
if self._custom_home:
self.ooni_home = self._custom_home
self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
self.decks_directory = os.path.join(self.ooni_home, 'decks')
self.reports_directory = os.path.join(self.ooni_home, 'reports')
self.report_log_file = os.path.join(self.ooni_home, 'reporting.yml')
self.resources_directory = os.path.join(self.data_directory, "resources")
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
self.config_file = expanduser(config_file)
else:
self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
if 'logfile' in self.basic:
self.basic.logfile = expanduser(self.basic.logfile.replace('~','~'+self.current_user))
if not os.path.exists(self.data_directory):
log.err("Data directory %s does not exists" % self.data_directory)
log.err("Edit data_dir inside of %s" % self.config_file)
def initialize_ooni_home(self, ooni_home=None):
if ooni_home:
self.set_paths(ooni_home)
if not os.path.isdir(self.ooni_home):
print "Ooni home directory does not exist."
print "Creating it in '%s'." % self.ooni_home
os.mkdir(self.ooni_home)
os.mkdir(self.inputs_directory)
os.mkdir(self.decks_directory)
if not os.path.isdir(self.reports_directory):
os.mkdir(self.reports_directory)
def _create_config_file(self):
target_config_file = self.config_file
print "Creating it for you in '%s'." % target_config_file
usr_share_path = '/usr/share'
if hasattr(sys, 'real_prefix'):
usr_share_path = os.path.abspath(os.path.join(sys.prefix, 'share'))
sample_config_file = os.path.join(self.data_directory,
'ooniprobe.conf.sample')
with open(sample_config_file) as f:
with open(target_config_file, 'w+') as w:
for line in f:
if line.startswith(' data_dir: '):
w.write(' data_dir: %s\n' % self.data_directory)
elif line.startswith(' geoip_data_dir: '):
w.write(' geoip_data_dir: %s\n' % os.path.join(usr_share_path, 'GeoIP'))
else:
w.write(line)
def read_config_file(self, check_incoherences=False):
if not os.path.isfile(self.config_file):
print "Configuration file does not exist."
self._create_config_file()
self.read_config_file()
with open(self.config_file) as f:
config_file_contents = '\n'.join(f.readlines())
configuration = yaml.safe_load(config_file_contents)
for setting in configuration.keys():
if setting in dir(self) and configuration[setting] is not None:
for k, v in configuration[setting].items():
getattr(self, setting)[k] = v
self.set_paths()
if check_incoherences:
self.check_incoherences(configuration)
def check_incoherences(self, configuration):
incoherent = []
if configuration['advanced']['interface'] != 'auto':
from scapy.all import get_if_list
if configuration['advanced']['interface'] not in get_if_list():
incoherent.append('advanced:interface')
self.log_incoherences(incoherent)
def log_incoherences(self, incoherences):
if len(incoherences) > 0:
if len(incoherences) > 1:
incoherent_pretty = ", ".join(incoherences[:-1]) + ' and ' + incoherences[-1]
else:
incoherent_pretty = incoherences[0]
log.err("You must set properly %s in %s." % (incoherent_pretty, self.config_file))
raise errors.ConfigFileIncoherent
@defer.inlineCallbacks
def check_tor(self):
"""
Called only when we must start tor by director.start
"""
incoherent = []
if not self.advanced.start_tor:
if self.tor.socks_port is None:
incoherent.append('tor:socks_port')
else:
socks_port_ep = TCP4ClientEndpoint(reactor,
"localhost",
self.tor.socks_port)
try:
yield connectProtocol(socks_port_ep, ConnectAndCloseProtocol())
except Exception:
incoherent.append('tor:socks_port')
if self.tor.control_port is not None:
control_port_ep = TCP4ClientEndpoint(reactor,
"localhost",
self.tor.control_port)
try:
yield connectProtocol(control_port_ep, ConnectAndCloseProtocol())
except Exception:
incoherent.append('tor:control_port')
self.log_incoherences(incoherent)
class OConfig(object):
_custom_home = None
def __init__(self):
self.current_user = getpass.getuser()
self.global_options = {}
self.reports = Storage()
self.scapyFactory = None
self.tor_state = None
# This is used to store the probes IP address obtained via Tor
self.probe_ip = geoip.ProbeIP()
# This is used to keep track of the state of the sniffer
self.sniffer_running = None
self.logging = True
self.basic = Storage()
self.advanced = Storage()
self.tor = Storage()
self.privacy = Storage()
self.set_paths()
def set_paths(self, ooni_home=None):
if ooni_home:
self._custom_home = ooni_home
if self.global_options.get('datadir'):
self.data_directory = abspath(expanduser(self.global_options['datadir']))
elif self.advanced.get('data_dir'):
self.data_directory = self.advanced['data_dir']
elif hasattr(sys, 'real_prefix'):
self.data_directory = os.path.abspath(os.path.join(sys.prefix, 'share', 'ooni'))
else:
self.data_directory = '/usr/share/ooni/'
self.nettest_directory = abspath(os.path.join(__file__, '..', 'nettests'))
self.ooni_home = os.path.join(expanduser('~'+self.current_user), '.ooni')
if self._custom_home:
self.ooni_home = self._custom_home
self.inputs_directory = os.path.join(self.ooni_home, 'inputs')
self.decks_directory = os.path.join(self.ooni_home, 'decks')
self.reports_directory = os.path.join(self.ooni_home, 'reports')
if self.global_options.get('configfile'):
config_file = self.global_options['configfile']
self.config_file = expanduser(config_file)
else:
self.config_file = os.path.join(self.ooni_home, 'ooniprobe.conf')
if 'logfile' in self.basic:
self.basic.logfile = expanduser(self.basic.logfile.replace('~','~'+self.current_user))
def initialize_ooni_home(self, ooni_home=None):
if ooni_home:
self.set_paths(ooni_home)
if not os.path.isdir(self.ooni_home):
print "Ooni home directory does not exist."
print "Creating it in '%s'." % self.ooni_home
os.mkdir(self.ooni_home)
os.mkdir(self.inputs_directory)
os.mkdir(self.decks_directory)
if not os.path.isdir(self.reports_directory):
os.mkdir(self.reports_directory)
def _create_config_file(self):
sample_config_file = os.path.join(self.data_directory,
'ooniprobe.conf.sample')
target_config_file = self.config_file
print "Creating it for you in '%s'." % target_config_file
usr_share_path = '/usr/share'
if hasattr(sys, 'real_prefix'):
usr_share_path = os.path.abspath(os.path.join(sys.prefix, 'share'))
with open(sample_config_file) as f:
with open(target_config_file, 'w+') as w:
for line in f:
if line.startswith(' data_dir: '):
w.write(' data_dir: %s\n' % os.path.join(usr_share_path, 'ooni'))
elif line.startswith(' geoip_data_dir: '):
w.write(' geoip_data_dir: %s\n' % os.path.join(usr_share_path, 'GeoIP'))
else:
w.write(line)
def read_config_file(self):
try:
with open(self.config_file) as f: pass
except IOError:
print "Configuration file does not exist."
self._create_config_file()
self.read_config_file()
with open(self.config_file) as f:
config_file_contents = '\n'.join(f.readlines())
configuration = yaml.safe_load(config_file_contents)
for setting in ['basic', 'reports', 'advanced', 'privacy', 'tor']:
try:
for k, v in configuration[setting].items():
getattr(self, setting)[k] = v
except AttributeError:
pass
self.set_paths()
def generate_pcap_filename(self, testDetails):
test_name, start_time = testDetails['test_name'], testDetails['start_time']
start_time = otime.epochToTimestamp(start_time)
return "report-%s-%s.%s" % (test_name, start_time, "pcap")
import os import yaml from twisted.internet import reactor, threads, defer from ooni import otime from ooni.utils import Storage scapyFactory = None stateDict = None state = Storage() # XXX refactor this to use a database resume_lock = defer.DeferredLock() basic = None cmd_line_options = None resume_filename = None # XXX-Twisted this is used to check if we have started the reactor or not. It # is necessary because if the tests are already concluded because we have # resumed a test session then it will call reactor.run() even though there is # no condition that will ever stop it. # There should be a more twisted way of doing this. start_reactor = True tor_state = None tor_control = None config_file = None sample_config_file = None
