def test_human_size(self):
self.assertEqual(
human_size_to_bytes("1G"),
1024**3
)
self.assertEqual(
human_size_to_bytes("1.3M"),
1.3 * 1024**2
)
self.assertEqual(
human_size_to_bytes("1.2K"),
1.2 * 1024
)
self.assertEqual(
human_size_to_bytes("1"),
1.0
)
self.assertEqual(
human_size_to_bytes("100.2"),
100.2
)
def task(self):
if config.basic.measurement_quota is None:
return
maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
used_bytes = directory_usage(config.measurements_directory)
warning_path = os.path.join(config.running_path, 'quota_warning')
if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when:
log.warn("You are about to reach the maximum allowed quota. Be careful")
with open(warning_path, "w") as out_file:
out_file.write("{0} {1}".format(used_bytes,
maximum_bytes))
else:
try:
os.remove(warning_path)
except OSError as ose:
if ose.errno != errno.ENOENT:
raise
if float(used_bytes) < float(maximum_bytes):
# We are within the allow quota exit.
return
# We should begin to delete old reports
amount_to_delete = float(used_bytes) - float(maximum_bytes)
amount_deleted = 0
measurement_path = FilePath(config.measurements_directory)
kept_measurements = []
stale_measurements = []
remaining_measurements = []
measurements_by_date = sorted(list_measurements(compute_size=True),
key=lambda k: k['test_start_time'])
for measurement in measurements_by_date:
if measurement['keep'] is True:
kept_measurements.append(measurement)
elif measurement['stale'] is True:
stale_measurements.append(measurement)
else:
remaining_measurements.append(measurement)
# This is the order in which we should begin deleting measurements.
ordered_measurements = (stale_measurements +
remaining_measurements +
kept_measurements)
while amount_deleted < amount_to_delete:
measurement = ordered_measurements.pop(0)
log.warn("Deleting report {0}".format(measurement["id"]))
measurement_path.child(measurement['id']).remove()
amount_deleted += measurement['size']
def start(self, logfile=None, application_name="ooniprobe"):
from ooni.settings import config
if not logfile:
logfile = os.path.expanduser(config.basic.logfile)
log_folder = os.path.dirname(logfile)
if (not os.access(log_folder, os.W_OK) or
(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
# If we don't have permissions to write to the log_folder or
# logfile.
log_folder = config.running_path
logfile = os.path.join(log_folder, "ooniprobe.log")
self.log_filepath = logfile
mkdir_p(log_folder)
log_filename = os.path.basename(logfile)
file_log_level = levels.get(config.basic.loglevel, levels['INFO'])
stdout_log_level = levels['INFO']
if config.advanced.debug:
stdout_log_level = levels['DEBUG']
if config.basic.rotate == 'daily':
logfile = MyDailyLogFile(log_filename, log_folder)
elif config.basic.rotate == 'length':
logfile = LogFile(log_filename,
log_folder,
rotateLength=int(
human_size_to_bytes(
config.basic.rotate_length)),
maxRotatedFiles=config.basic.max_rotated_files)
else:
logfile = open(os.path.join(log_folder, log_filename), 'a')
self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level)
self.stdoutObserver = StdoutStderrObserver(sys.stdout,
log_level=stdout_log_level)
tw_log.startLoggingWithObserver(self.fileObserver.emit)
tw_log.addObserver(self.stdoutObserver.emit)
tw_log.msg("Starting %s on %s (%s UTC)" %
(application_name, otime.prettyDateNow(),
otime.prettyDateNowUTC()))
def start(self, logfile=None, application_name="ooniprobe"):
from ooni.settings import config
if not logfile:
logfile = os.path.expanduser(config.basic.logfile)
log_folder = os.path.dirname(logfile)
if (not os.access(log_folder, os.W_OK) or
(os.path.exists(logfile) and not os.access(logfile, os.W_OK))):
# If we don't have permissions to write to the log_folder or
# logfile.
log_folder = config.running_path
logfile = os.path.join(log_folder, "ooniprobe.log")
self.log_filepath = logfile
mkdir_p(log_folder)
log_filename = os.path.basename(logfile)
file_log_level = levels.get(config.basic.loglevel,
levels['INFO'])
stdout_log_level = levels['INFO']
if config.advanced.debug:
stdout_log_level = levels['DEBUG']
if config.basic.rotate == 'daily':
logfile = MyDailyLogFile(log_filename, log_folder)
elif config.basic.rotate == 'length':
logfile = LogFile(log_filename, log_folder,
rotateLength=int(human_size_to_bytes(
config.basic.rotate_length
)),
maxRotatedFiles=config.basic.max_rotated_files)
else:
logfile = open(os.path.join(log_folder, log_filename), 'a')
self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level)
self.stdoutObserver = StdoutStderrObserver(sys.stdout,
log_level=stdout_log_level)
tw_log.startLoggingWithObserver(self.fileObserver.emit)
tw_log.addObserver(self.stdoutObserver.emit)
tw_log.msg("Starting %s on %s (%s UTC)" % (application_name,
otime.prettyDateNow(),
otime.prettyDateNowUTC()))
def test_human_size(self):
self.assertEqual(human_size_to_bytes("1G"), 1024 ** 3)
self.assertEqual(human_size_to_bytes("1.3M"), 1.3 * 1024 ** 2)
self.assertEqual(human_size_to_bytes("1.2K"), 1.2 * 1024)
self.assertEqual(human_size_to_bytes("1"), 1.0)
self.assertEqual(human_size_to_bytes("100.2"), 100.2)
