Example #1
0
 def test_directory_usage(self):
     tmp_dir = tempfile.mkdtemp()
     with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file:
         out_file.write("A" * 1000)
     os.mkdir(os.path.join(tmp_dir, "subdir"))
     with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file:
         out_file.write("A" * 1000)
     self.assertEqual(directory_usage(tmp_dir), 1000 * 2)
Example #2
0
 def test_directory_usage(self):
     tmp_dir = tempfile.mkdtemp()
     with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file:
         out_file.write("A"*1000)
     os.mkdir(os.path.join(tmp_dir, "subdir"))
     with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file:
         out_file.write("A"*1000)
     self.assertEqual(directory_usage(tmp_dir), 1000*2)
Example #3
0
def get_measurement(measurement_id, compute_size=False):
    size = -1
    measurement_path = FilePath(config.measurements_directory)
    measurement = measurement_path.child(measurement_id)
    if not measurement.exists():
        raise MeasurementNotFound

    running = False
    completed = True
    keep = False
    stale = False
    anomaly = False
    if measurement.child("measurements.njson.progress").exists():
        completed = False
        try:
            pid = measurement.child("running.pid").open("r").read()
            pid = int(pid)
            if is_process_running(pid):
                running = True
            else:
                stale = True
        except IOError:
            stale = True

    if measurement.child("keep").exists():
        keep = True

    if measurement.child("anomaly").exists():
        anomaly = True

    if compute_size is True:
        size = directory_usage(measurement.path)

    measurement_metadata = measurement_id.split("-")
    test_start_time, country_code, asn, test_name = measurement_metadata[:4]
    deck_id = "none"
    if len(measurement_metadata) > 4:
        deck_id = '-'.join(measurement_metadata[4:])
    return {
        "test_name": test_name,
        "country_code": country_code,
        "asn": asn,
        "test_start_time": test_start_time,
        "id": measurement_id,
        "completed": completed,
        "keep": keep,
        "running": running,
        "stale": stale,
        "size": size,
        "deck_id": deck_id,
        "anomaly": anomaly
    }
Example #4
0
    def task(self):
        if config.basic.measurement_quota is None:
            return
        maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
        used_bytes = directory_usage(config.measurements_directory)
        warning_path = os.path.join(config.running_path, 'quota_warning')

        if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when:
            log.warn("You are about to reach the maximum allowed quota. Be careful")
            with open(warning_path, "w") as out_file:
                out_file.write("{0} {1}".format(used_bytes,
                                                maximum_bytes))
        else:
            try:
                os.remove(warning_path)
            except OSError as ose:
                if ose.errno != errno.ENOENT:
                    raise

        if float(used_bytes) < float(maximum_bytes):
            # We are within the allow quota exit.
            return

        # We should begin to delete old reports
        amount_to_delete = float(used_bytes) - float(maximum_bytes)
        amount_deleted = 0
        measurement_path = FilePath(config.measurements_directory)

        kept_measurements = []
        stale_measurements = []
        remaining_measurements = []
        measurements_by_date = sorted(list_measurements(compute_size=True),
                                      key=lambda k: k['test_start_time'])
        for measurement in measurements_by_date:
            if measurement['keep'] is True:
                kept_measurements.append(measurement)
            elif measurement['stale'] is True:
                stale_measurements.append(measurement)
            else:
                remaining_measurements.append(measurement)

        # This is the order in which we should begin deleting measurements.
        ordered_measurements = (stale_measurements +
                                remaining_measurements +
                                kept_measurements)
        while amount_deleted < amount_to_delete:
            measurement = ordered_measurements.pop(0)
            log.warn("Deleting report {0}".format(measurement["id"]))
            measurement_path.child(measurement['id']).remove()
            amount_deleted += measurement['size']
Example #5
0
def get_measurement(measurement_id, compute_size=False):
    size = -1
    measurement_path = FilePath(config.measurements_directory)
    measurement = measurement_path.child(measurement_id)
    if not measurement.exists():
        raise MeasurementNotFound

    running = False
    completed = True
    keep = False
    stale = False
    if measurement.child("measurements.njson.progress").exists():
        completed = False
        pid = measurement.child("running.pid").open("r").read()
        pid = int(pid)
        if is_process_running(pid):
            running = True
        else:
            stale = True

    if measurement.child("keep").exists():
        keep = True

    if compute_size is True:
        size = directory_usage(measurement.path)

    test_start_time, country_code, asn, test_name = \
        measurement_id.split("-")[:4]
    return {
        "test_name": test_name,
        "country_code": country_code,
        "asn": asn,
        "test_start_time": test_start_time,
        "id": measurement_id,
        "completed": completed,
        "keep": keep,
        "running": running,
        "stale": stale,
        "size": size
    }