def test_human_size(self): self.assertEqual( human_size_to_bytes("1G"), 1024**3 ) self.assertEqual( human_size_to_bytes("1.3M"), 1.3 * 1024**2 ) self.assertEqual( human_size_to_bytes("1.2K"), 1.2 * 1024 ) self.assertEqual( human_size_to_bytes("1"), 1.0 ) self.assertEqual( human_size_to_bytes("100.2"), 100.2 )
def task(self): if config.basic.measurement_quota is None: return maximum_bytes = human_size_to_bytes(config.basic.measurement_quota) used_bytes = directory_usage(config.measurements_directory) warning_path = os.path.join(config.running_path, 'quota_warning') if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when: log.warn("You are about to reach the maximum allowed quota. Be careful") with open(warning_path, "w") as out_file: out_file.write("{0} {1}".format(used_bytes, maximum_bytes)) else: try: os.remove(warning_path) except OSError as ose: if ose.errno != errno.ENOENT: raise if float(used_bytes) < float(maximum_bytes): # We are within the allow quota exit. return # We should begin to delete old reports amount_to_delete = float(used_bytes) - float(maximum_bytes) amount_deleted = 0 measurement_path = FilePath(config.measurements_directory) kept_measurements = [] stale_measurements = [] remaining_measurements = [] measurements_by_date = sorted(list_measurements(compute_size=True), key=lambda k: k['test_start_time']) for measurement in measurements_by_date: if measurement['keep'] is True: kept_measurements.append(measurement) elif measurement['stale'] is True: stale_measurements.append(measurement) else: remaining_measurements.append(measurement) # This is the order in which we should begin deleting measurements. ordered_measurements = (stale_measurements + remaining_measurements + kept_measurements) while amount_deleted < amount_to_delete: measurement = ordered_measurements.pop(0) log.warn("Deleting report {0}".format(measurement["id"])) measurement_path.child(measurement['id']).remove() amount_deleted += measurement['size']
def start(self, logfile=None, application_name="ooniprobe"): from ooni.settings import config if not logfile: logfile = os.path.expanduser(config.basic.logfile) log_folder = os.path.dirname(logfile) if (not os.access(log_folder, os.W_OK) or (os.path.exists(logfile) and not os.access(logfile, os.W_OK))): # If we don't have permissions to write to the log_folder or # logfile. log_folder = config.running_path logfile = os.path.join(log_folder, "ooniprobe.log") self.log_filepath = logfile mkdir_p(log_folder) log_filename = os.path.basename(logfile) file_log_level = levels.get(config.basic.loglevel, levels['INFO']) stdout_log_level = levels['INFO'] if config.advanced.debug: stdout_log_level = levels['DEBUG'] if config.basic.rotate == 'daily': logfile = MyDailyLogFile(log_filename, log_folder) elif config.basic.rotate == 'length': logfile = LogFile(log_filename, log_folder, rotateLength=int( human_size_to_bytes( config.basic.rotate_length)), maxRotatedFiles=config.basic.max_rotated_files) else: logfile = open(os.path.join(log_folder, log_filename), 'a') self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level) self.stdoutObserver = StdoutStderrObserver(sys.stdout, log_level=stdout_log_level) tw_log.startLoggingWithObserver(self.fileObserver.emit) tw_log.addObserver(self.stdoutObserver.emit) tw_log.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(), otime.prettyDateNowUTC()))
def start(self, logfile=None, application_name="ooniprobe"): from ooni.settings import config if not logfile: logfile = os.path.expanduser(config.basic.logfile) log_folder = os.path.dirname(logfile) if (not os.access(log_folder, os.W_OK) or (os.path.exists(logfile) and not os.access(logfile, os.W_OK))): # If we don't have permissions to write to the log_folder or # logfile. log_folder = config.running_path logfile = os.path.join(log_folder, "ooniprobe.log") self.log_filepath = logfile mkdir_p(log_folder) log_filename = os.path.basename(logfile) file_log_level = levels.get(config.basic.loglevel, levels['INFO']) stdout_log_level = levels['INFO'] if config.advanced.debug: stdout_log_level = levels['DEBUG'] if config.basic.rotate == 'daily': logfile = MyDailyLogFile(log_filename, log_folder) elif config.basic.rotate == 'length': logfile = LogFile(log_filename, log_folder, rotateLength=int(human_size_to_bytes( config.basic.rotate_length )), maxRotatedFiles=config.basic.max_rotated_files) else: logfile = open(os.path.join(log_folder, log_filename), 'a') self.fileObserver = MsecLogObserver(logfile, log_level=file_log_level) self.stdoutObserver = StdoutStderrObserver(sys.stdout, log_level=stdout_log_level) tw_log.startLoggingWithObserver(self.fileObserver.emit) tw_log.addObserver(self.stdoutObserver.emit) tw_log.msg("Starting %s on %s (%s UTC)" % (application_name, otime.prettyDateNow(), otime.prettyDateNowUTC()))
def test_human_size(self): self.assertEqual(human_size_to_bytes("1G"), 1024 ** 3) self.assertEqual(human_size_to_bytes("1.3M"), 1.3 * 1024 ** 2) self.assertEqual(human_size_to_bytes("1.2K"), 1.2 * 1024) self.assertEqual(human_size_to_bytes("1"), 1.0) self.assertEqual(human_size_to_bytes("100.2"), 100.2)