def _do_cleanup(self):
while not self._should_stop_cleanup():
while True:
# If the total size of all dependencies exceeds
# self._max_work_dir_size_bytes, remove the oldest unused
# dependency. Otherwise, break out of the loop.
total_size_bytes = 0
first_used_time = float('inf')
first_used_target = None
self._lock.acquire()
for target, dependency in self._dependencies.items():
if dependency.downloading:
continue
# We compute the size of dependencies here to keep the code
# that adds new bundles to the dependency manager simpler.
if dependency.size_bytes is None:
self._lock.release()
size_bytes = get_path_size(
os.path.join(self._bundles_dir, dependency.path))
self._lock.acquire()
dependency.size_bytes = size_bytes
self._save_state()
total_size_bytes += dependency.size_bytes
if (not dependency.has_children()
and dependency.last_used < first_used_time):
first_used_time = dependency.last_used
first_used_target = target
self._lock.release()
if (total_size_bytes > self._max_work_dir_size_bytes
and first_used_target is not None):
logger.info(
'used ({}) exceeds capacity ({}), removing oldest bundle from cache'
.format(size_str(total_size_bytes),
size_str(self._max_work_dir_size_bytes)))
with self._lock:
dependency = self._dependencies[first_used_target]
if dependency.has_children():
# Since we released the lock there could be new
# children.
continue
del self._dependencies[first_used_target]
self._paths.remove(dependency.path)
self._save_state()
remove_path(
os.path.join(self._bundles_dir, dependency.path))
else:
break
# Sleep for 10 seconds, allowing interruptions every second.
for _ in xrange(0, self._cleanup_sleep_secs):
time.sleep(1)
if self._should_stop_cleanup():
break
def update(self):
start_time = time.time()
try:
disk_utilization = get_path_size(self._bundle_path, self._exclude_paths)
# Setting the instance variable is atomic, so no need to lock
self._disk_utilization = disk_utilization
except Exception:
logging.exception('Problem calculating disk utilization for path %s.', self._bundle_path)
end_time = time.time()
self._start_time = start_time
self._end_time = end_time
return self
def _do_cleanup(self):
while not self._should_stop_cleanup():
while True:
# If the total size of all dependencies exceeds
# self._max_work_dir_size_mb, remove the oldest unused
# dependency. Otherwise, break out of the loop.
total_size_bytes = 0
first_used_time = float('inf')
first_used_target = None
self._lock.acquire()
for target, dependency in self._dependencies.items():
if dependency.downloading:
continue
# We compute the size of dependencies here to keep the code
# that adds new bundles to the dependency manager simpler.
if dependency.size_bytes is None:
self._lock.release()
size_bytes = get_path_size(os.path.join(self._work_dir,
dependency.path))
self._lock.acquire()
dependency.size_bytes = size_bytes
self._save_state()
total_size_bytes += dependency.size_bytes
if (not dependency.has_children() and
dependency.last_used < first_used_time):
first_used_time = dependency.last_used
first_used_target = target
self._lock.release()
if (total_size_bytes > self._max_work_dir_size_mb * 1024 * 1024 and
first_used_target is not None):
with self._lock:
dependency = self._dependencies[first_used_target]
if dependency.has_children():
# Since we released the lock there could be new
# children.
continue
del self._dependencies[first_used_target]
self._paths.remove(dependency.path)
self._save_state()
remove_path(os.path.join(self._work_dir, dependency.path))
else:
break
# Sleep for 10 seconds, allowing interruptions every second.
for _ in xrange(0, self._cleanup_sleep_secs):
time.sleep(1)
if self._should_stop_cleanup():
break
def _compute_disk_utilization(self):
while not self._is_finished():
start_time = time.time()
try:
disk_utilization = get_path_size(self._bundle_path, self._dep_paths)
with self._disk_utilization_lock:
self._disk_utilization = disk_utilization
except Exception:
traceback.print_exc()
end_time = time.time()
# To ensure that we don't hammer the disk for this computation when
# there are lots of files, we run it at most 10% of the time.
time.sleep(max((end_time - start_time) * 10, 1.0))
def _compute_disk_utilization(self):
while not self._is_finished():
start_time = time.time()
try:
disk_utilization = get_path_size(self._bundle_path, self._dep_paths)
with self._disk_utilization_lock:
self._disk_utilization = disk_utilization
except Exception:
traceback.print_exc()
end_time = time.time()
# To ensure that we don't hammer the disk for this computation when
# there are lots of files, we run it at most 10% of the time.
time.sleep(max((end_time - start_time) * 10, 1.0))