def test_directory_usage(self): tmp_dir = tempfile.mkdtemp() with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file: out_file.write("A" * 1000) os.mkdir(os.path.join(tmp_dir, "subdir")) with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file: out_file.write("A" * 1000) self.assertEqual(directory_usage(tmp_dir), 1000 * 2)
def test_directory_usage(self): tmp_dir = tempfile.mkdtemp() with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file: out_file.write("A"*1000) os.mkdir(os.path.join(tmp_dir, "subdir")) with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file: out_file.write("A"*1000) self.assertEqual(directory_usage(tmp_dir), 1000*2)
def get_measurement(measurement_id, compute_size=False): size = -1 measurement_path = FilePath(config.measurements_directory) measurement = measurement_path.child(measurement_id) if not measurement.exists(): raise MeasurementNotFound running = False completed = True keep = False stale = False anomaly = False if measurement.child("measurements.njson.progress").exists(): completed = False try: pid = measurement.child("running.pid").open("r").read() pid = int(pid) if is_process_running(pid): running = True else: stale = True except IOError: stale = True if measurement.child("keep").exists(): keep = True if measurement.child("anomaly").exists(): anomaly = True if compute_size is True: size = directory_usage(measurement.path) measurement_metadata = measurement_id.split("-") test_start_time, country_code, asn, test_name = measurement_metadata[:4] deck_id = "none" if len(measurement_metadata) > 4: deck_id = '-'.join(measurement_metadata[4:]) return { "test_name": test_name, "country_code": country_code, "asn": asn, "test_start_time": test_start_time, "id": measurement_id, "completed": completed, "keep": keep, "running": running, "stale": stale, "size": size, "deck_id": deck_id, "anomaly": anomaly }
def task(self): if config.basic.measurement_quota is None: return maximum_bytes = human_size_to_bytes(config.basic.measurement_quota) used_bytes = directory_usage(config.measurements_directory) warning_path = os.path.join(config.running_path, 'quota_warning') if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when: log.warn("You are about to reach the maximum allowed quota. Be careful") with open(warning_path, "w") as out_file: out_file.write("{0} {1}".format(used_bytes, maximum_bytes)) else: try: os.remove(warning_path) except OSError as ose: if ose.errno != errno.ENOENT: raise if float(used_bytes) < float(maximum_bytes): # We are within the allow quota exit. return # We should begin to delete old reports amount_to_delete = float(used_bytes) - float(maximum_bytes) amount_deleted = 0 measurement_path = FilePath(config.measurements_directory) kept_measurements = [] stale_measurements = [] remaining_measurements = [] measurements_by_date = sorted(list_measurements(compute_size=True), key=lambda k: k['test_start_time']) for measurement in measurements_by_date: if measurement['keep'] is True: kept_measurements.append(measurement) elif measurement['stale'] is True: stale_measurements.append(measurement) else: remaining_measurements.append(measurement) # This is the order in which we should begin deleting measurements. ordered_measurements = (stale_measurements + remaining_measurements + kept_measurements) while amount_deleted < amount_to_delete: measurement = ordered_measurements.pop(0) log.warn("Deleting report {0}".format(measurement["id"])) measurement_path.child(measurement['id']).remove() amount_deleted += measurement['size']
def get_measurement(measurement_id, compute_size=False): size = -1 measurement_path = FilePath(config.measurements_directory) measurement = measurement_path.child(measurement_id) if not measurement.exists(): raise MeasurementNotFound running = False completed = True keep = False stale = False if measurement.child("measurements.njson.progress").exists(): completed = False pid = measurement.child("running.pid").open("r").read() pid = int(pid) if is_process_running(pid): running = True else: stale = True if measurement.child("keep").exists(): keep = True if compute_size is True: size = directory_usage(measurement.path) test_start_time, country_code, asn, test_name = \ measurement_id.split("-")[:4] return { "test_name": test_name, "country_code": country_code, "asn": asn, "test_start_time": test_start_time, "id": measurement_id, "completed": completed, "keep": keep, "running": running, "stale": stale, "size": size }