def get_tor_config():
tor_config = TorConfig()
if config.tor.control_port is None:
config.tor.control_port = int(randomFreePort())
if config.tor.socks_port is None:
config.tor.socks_port = int(randomFreePort())
tor_config.ControlPort = config.tor.control_port
tor_config.SocksPort = config.tor.socks_port
if config.tor.data_dir:
data_dir = os.path.expanduser(config.tor.data_dir)
# We only use the Tor data dir specified in the config file if
# 1. It is not locked (i.e. another process is using it)
# 2. We have write permissions to it
data_dir_usable = is_tor_data_dir_usable(data_dir)
try:
mkdir_p(data_dir)
except OSError as ose:
if ose.errno == errno.EACCESS:
data_dir_usable = False
else:
raise
if data_dir_usable:
tor_config.DataDirectory = data_dir
if config.tor.bridges:
tor_config.UseBridges = 1
if config.advanced.obfsproxy_binary:
tor_config.ClientTransportPlugin = (
'obfs2,obfs3 exec %s managed' %
config.advanced.obfsproxy_binary
)
bridges = []
with open(config.tor.bridges) as f:
for bridge in f:
if 'obfs' in bridge:
if config.advanced.obfsproxy_binary:
bridges.append(bridge.strip())
else:
bridges.append(bridge.strip())
tor_config.Bridge = bridges
if config.tor.torrc:
for i in config.tor.torrc.keys():
setattr(tor_config, i, config.tor.torrc[i])
if os.geteuid() == 0:
tor_config.User = pwd.getpwuid(os.geteuid()).pw_name
tor_config.save()
log.debug("Setting control port as %s" % tor_config.ControlPort)
log.debug("Setting SOCKS port as %s" % tor_config.SocksPort)
return tor_config
def create(self, country_code=None):
# XXX This is a hax to avoid race conditions in testing because this
# object is a singleton and config can have a custom home directory
# passed at runtime.
self.path = FilePath(config.inputs_directory)
self.resources = FilePath(config.resources_directory)
mkdir_p(self.path.child("descriptors").path)
mkdir_p(self.path.child("data").path)
yield self.update_url_lists(country_code)
yield self.update_tor_bridge_lines(country_code)
def create(self, country_code=None):
# XXX This is a hax to avoid race conditions in testing because this
# object is a singleton and config can have a custom home directory
# passed at runtime.
self.path = FilePath(config.inputs_directory)
self.resources = FilePath(config.resources_directory)
mkdir_p(self.path.child("descriptors").path)
mkdir_p(self.path.child("data").path)
yield self.update_url_lists(country_code)
yield self.update_tor_bridge_lines(country_code)
def start(self, logfile=None, application_name="ooniprobe"):
from ooni.settings import config
if not logfile:
logfile = os.path.expanduser(config.basic.logfile)
log_folder = os.path.dirname(logfile)
if (not os.access(log_folder, os.W_OK) or
(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
# If we don't have permissions to write to the log_folder or
# logfile.
log_folder = config.running_path
logfile = os.path.join(log_folder, "ooniprobe.log")
self.log_filepath = logfile
mkdir_p(log_folder)
log_filename = os.path.basename(logfile)
file_log_level = levels.get(config.basic.loglevel, levels['INFO'])
stdout_log_level = levels['INFO']
if config.advanced.debug:
stdout_log_level = levels['DEBUG']
if config.basic.rotate == 'daily':
logfile = MyDailyLogFile(log_filename, log_folder)
elif config.basic.rotate == 'length':
logfile = LogFile(log_filename,
log_folder,
rotateLength=int(
human_size_to_bytes(
config.basic.rotate_length)),
maxRotatedFiles=config.basic.max_rotated_files)
else:
logfile = open(os.path.join(log_folder, log_filename), 'a')
self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level)
self.stdoutObserver = StdoutStderrObserver(sys.stdout,
log_level=stdout_log_level)
tw_log.startLoggingWithObserver(self.fileObserver.emit)
tw_log.addObserver(self.stdoutObserver.emit)
tw_log.msg("Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(),
otime.prettyDateNowUTC()))
def start(self, logfile=None, application_name="ooniprobe"):
from ooni.settings import config
if not logfile:
logfile = os.path.expanduser(config.basic.logfile)
log_folder = os.path.dirname(logfile)
if (not os.access(log_folder, os.W_OK) or
(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
# If we don't have permissions to write to the log_folder or
# logfile.
log_folder = config.running_path
logfile = os.path.join(log_folder, "ooniprobe.log")
self.log_filepath = logfile
mkdir_p(log_folder)
log_filename = os.path.basename(logfile)
file_log_level = levels.get(config.basic.loglevel,
levels['INFO'])
stdout_log_level = levels['INFO']
if config.advanced.debug:
stdout_log_level = levels['DEBUG']
if config.basic.rotate == 'daily':
logfile = MyDailyLogFile(log_filename, log_folder)
elif config.basic.rotate == 'length':
logfile = LogFile(log_filename, log_folder,
rotateLength=int(human_size_to_bytes(
config.basic.rotate_length
)),
maxRotatedFiles=config.basic.max_rotated_files)
else:
logfile = open(os.path.join(log_folder, log_filename), 'a')
self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level)
self.stdoutObserver = StdoutStderrObserver(sys.stdout,
log_level=stdout_log_level)
tw_log.startLoggingWithObserver(self.fileObserver.emit)
tw_log.addObserver(self.stdoutObserver.emit)
tw_log.msg("Starting %s on %s (%s UTC)" % (application_name,
otime.prettyDateNow(),
otime.prettyDateNowUTC()))
def oonideckgen(reactor):
options = Options()
try:
options.parseOptions()
except usage.UsageError as error_message:
print("%s: %s" % (sys.argv[0], error_message))
print(options)
sys.exit(1)
print("Checking for update of resources")
yield check_for_update()
if not options['output']:
options['output'] = os.getcwd()
if not options['country-code']:
try:
options['country-code'] = yield get_user_country_code()
except errors.ProbeIPUnknown:
print("Could not determine your IP address.")
print("Check your internet connection or specify a country code "
"with -c.")
sys.exit(4)
if len(options['country-code']) != 2:
print("%s: --country-code must be 2 characters" % sys.argv[0])
sys.exit(2)
if os.path.isdir(options['output']):
options['output'] = os.path.join(options['output'], 'web-full.yaml')
options['country-code'] = options['country-code'].lower()
mkdir_p(os.path.dirname(options['output']))
generate_deck(options)
def oonideckgen(reactor):
options = Options()
try:
options.parseOptions()
except usage.UsageError as error_message:
print("%s: %s" % (sys.argv[0], error_message))
print(options)
sys.exit(1)
print("Checking for update of resources")
yield check_for_update()
if not options['output']:
options['output'] = os.getcwd()
if not options['country-code']:
try:
options['country-code'] = yield get_user_country_code()
except errors.ProbeIPUnknown:
print("Could not determine your IP address.")
print("Check your internet connection or specify a country code "
"with -c.")
sys.exit(4)
if len(options['country-code']) != 2:
print("%s: --country-code must be 2 characters" % sys.argv[0])
sys.exit(2)
if os.path.isdir(options['output']):
options['output'] = os.path.join(options['output'], 'web-full.yaml')
options['country-code'] = options['country-code'].lower()
mkdir_p(os.path.dirname(options['output']))
generate_deck(options)
def check_for_update(country_code=None):
"""
Checks if we need to update the resources.
If the country_code is specified then only the resources for that
country will be updated/downloaded.
XXX we currently don't check the shasum of resources although this is
included inside of the manifest.
This should probably be done once we have signing of resources.
:return: the latest version.
"""
temporary_files = []
def cleanup():
# If we fail we need to delete all the temporary files
for _, src_file_path in temporary_files:
src_file_path.remove()
current_version = get_current_version()
latest_version = yield get_latest_version()
resources_dir = FilePath(config.resources_directory)
mkdir_p(resources_dir.path)
current_manifest = resources_dir.child("manifest.json")
if current_manifest.exists():
with current_manifest.open("r") as f:
current_manifest_data = json.load(f)
else:
current_manifest_data = {"resources": []}
# We should download a newer manifest
if current_version < latest_version:
new_manifest = current_manifest.temporarySibling()
new_manifest.alwaysCreate = 0
temporary_files.append((current_manifest, new_manifest))
try:
yield downloadPage(
get_download_url(latest_version, "manifest.json"),
new_manifest.path)
except:
cleanup()
raise UpdateFailure("Failed to download manifest")
new_manifest_data = json.loads(new_manifest.getContent())
else:
new_manifest_data = current_manifest_data
to_update, to_delete = get_out_of_date_resources(current_manifest_data,
new_manifest_data,
country_code)
try:
for resource in to_update:
gzipped = False
pre_path, filename = resource["path"].split("/")
if filename.endswith(".gz"):
filename = filename[:-3]
gzipped = True
dst_file = resources_dir.child(pre_path).child(filename)
mkdir_p(dst_file.parent().path)
src_file = dst_file.temporarySibling()
src_file.alwaysCreate = 0
temporary_files.append((dst_file, src_file))
# The paths for the download require replacing "/" with "."
download_url = get_download_url(latest_version,
resource["path"].replace("/", "."))
yield downloadPage(download_url, src_file.path)
if gzipped:
gunzip(src_file.path)
except Exception as exc:
cleanup()
log.exception(exc)
raise UpdateFailure("Failed to download resource {0}".format(
resource["path"]))
for dst_file, src_file in temporary_files:
log.msg("Moving {0} to {1}".format(src_file.path, dst_file.path))
rename(src_file.path, dst_file.path)
for resource in to_delete:
log.msg("Deleting old resources")
pre_path, filename = resource["path"].split("/")
resources_dir.child(pre_path).child(filename).remove()
def check_for_update(country_code=None):
"""
Checks if we need to update the resources.
If the country_code is specified then only the resources for that
country will be updated/downloaded.
XXX we currently don't check the shasum of resources although this is
included inside of the manifest.
This should probably be done once we have signing of resources.
:return: the latest version.
"""
temporary_files = []
def cleanup():
# If we fail we need to delete all the temporary files
for _, src_file_path in temporary_files:
src_file_path.remove()
current_version = get_current_version()
latest_version = yield get_latest_version()
resources_dir = FilePath(config.resources_directory)
mkdir_p(resources_dir.path)
current_manifest = resources_dir.child("manifest.json")
if current_manifest.exists():
with current_manifest.open("r") as f:
current_manifest_data = json.load(f)
else:
current_manifest_data = {
"resources": []
}
# We should download a newer manifest
if current_version < latest_version:
new_manifest = current_manifest.temporarySibling()
new_manifest.alwaysCreate = 0
temporary_files.append((current_manifest, new_manifest))
try:
yield downloadPage(
get_download_url(latest_version, "manifest.json"),
new_manifest.path
)
except:
cleanup()
raise UpdateFailure("Failed to download manifest")
new_manifest_data = json.loads(new_manifest.getContent())
else:
new_manifest_data = current_manifest_data
to_update, to_delete = get_out_of_date_resources(
current_manifest_data, new_manifest_data, country_code)
try:
for resource in to_update:
gzipped = False
pre_path, filename = resource["path"].split("/")
if filename.endswith(".gz"):
filename = filename[:-3]
gzipped = True
dst_file = resources_dir.child(pre_path).child(filename)
mkdir_p(dst_file.parent().path)
src_file = dst_file.temporarySibling()
src_file.alwaysCreate = 0
temporary_files.append((dst_file, src_file))
# The paths for the download require replacing "/" with "."
download_url = get_download_url(latest_version,
resource["path"].replace("/", "."))
yield downloadPage(download_url, src_file.path)
if gzipped:
gunzip(src_file.path)
except Exception as exc:
cleanup()
log.exception(exc)
raise UpdateFailure("Failed to download resource {0}".format(resource["path"]))
for dst_file, src_file in temporary_files:
log.msg("Moving {0} to {1}".format(src_file.path,
dst_file.path))
rename(src_file.path, dst_file.path)
for resource in to_delete:
log.msg("Deleting old resources")
resources_dir.child(resource["path"]).remove()