def test_directory_usage(self):
tmp_dir = tempfile.mkdtemp()
with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file:
out_file.write("A" * 1000)
os.mkdir(os.path.join(tmp_dir, "subdir"))
with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file:
out_file.write("A" * 1000)
self.assertEqual(directory_usage(tmp_dir), 1000 * 2)
def test_directory_usage(self):
tmp_dir = tempfile.mkdtemp()
with open(os.path.join(tmp_dir, "something.txt"), "w") as out_file:
out_file.write("A"*1000)
os.mkdir(os.path.join(tmp_dir, "subdir"))
with open(os.path.join(tmp_dir, "subdir", "something.txt"), "w") as out_file:
out_file.write("A"*1000)
self.assertEqual(directory_usage(tmp_dir), 1000*2)
def get_measurement(measurement_id, compute_size=False):
size = -1
measurement_path = FilePath(config.measurements_directory)
measurement = measurement_path.child(measurement_id)
if not measurement.exists():
raise MeasurementNotFound
running = False
completed = True
keep = False
stale = False
anomaly = False
if measurement.child("measurements.njson.progress").exists():
completed = False
try:
pid = measurement.child("running.pid").open("r").read()
pid = int(pid)
if is_process_running(pid):
running = True
else:
stale = True
except IOError:
stale = True
if measurement.child("keep").exists():
keep = True
if measurement.child("anomaly").exists():
anomaly = True
if compute_size is True:
size = directory_usage(measurement.path)
measurement_metadata = measurement_id.split("-")
test_start_time, country_code, asn, test_name = measurement_metadata[:4]
deck_id = "none"
if len(measurement_metadata) > 4:
deck_id = '-'.join(measurement_metadata[4:])
return {
"test_name": test_name,
"country_code": country_code,
"asn": asn,
"test_start_time": test_start_time,
"id": measurement_id,
"completed": completed,
"keep": keep,
"running": running,
"stale": stale,
"size": size,
"deck_id": deck_id,
"anomaly": anomaly
}
def task(self):
if config.basic.measurement_quota is None:
return
maximum_bytes = human_size_to_bytes(config.basic.measurement_quota)
used_bytes = directory_usage(config.measurements_directory)
warning_path = os.path.join(config.running_path, 'quota_warning')
if (float(used_bytes) / float(maximum_bytes)) >= self._warn_when:
log.warn("You are about to reach the maximum allowed quota. Be careful")
with open(warning_path, "w") as out_file:
out_file.write("{0} {1}".format(used_bytes,
maximum_bytes))
else:
try:
os.remove(warning_path)
except OSError as ose:
if ose.errno != errno.ENOENT:
raise
if float(used_bytes) < float(maximum_bytes):
# We are within the allow quota exit.
return
# We should begin to delete old reports
amount_to_delete = float(used_bytes) - float(maximum_bytes)
amount_deleted = 0
measurement_path = FilePath(config.measurements_directory)
kept_measurements = []
stale_measurements = []
remaining_measurements = []
measurements_by_date = sorted(list_measurements(compute_size=True),
key=lambda k: k['test_start_time'])
for measurement in measurements_by_date:
if measurement['keep'] is True:
kept_measurements.append(measurement)
elif measurement['stale'] is True:
stale_measurements.append(measurement)
else:
remaining_measurements.append(measurement)
# This is the order in which we should begin deleting measurements.
ordered_measurements = (stale_measurements +
remaining_measurements +
kept_measurements)
while amount_deleted < amount_to_delete:
measurement = ordered_measurements.pop(0)
log.warn("Deleting report {0}".format(measurement["id"]))
measurement_path.child(measurement['id']).remove()
amount_deleted += measurement['size']
def get_measurement(measurement_id, compute_size=False):
size = -1
measurement_path = FilePath(config.measurements_directory)
measurement = measurement_path.child(measurement_id)
if not measurement.exists():
raise MeasurementNotFound
running = False
completed = True
keep = False
stale = False
if measurement.child("measurements.njson.progress").exists():
completed = False
pid = measurement.child("running.pid").open("r").read()
pid = int(pid)
if is_process_running(pid):
running = True
else:
stale = True
if measurement.child("keep").exists():
keep = True
if compute_size is True:
size = directory_usage(measurement.path)
test_start_time, country_code, asn, test_name = \
measurement_id.split("-")[:4]
return {
"test_name": test_name,
"country_code": country_code,
"asn": asn,
"test_start_time": test_start_time,
"id": measurement_id,
"completed": completed,
"keep": keep,
"running": running,
"stale": stale,
"size": size
}