def check_disk_utilization(run_state): try: disk_utilization = get_path_size(run_state.bundle_path) run_state._replace(disk_utilization=disk_utilization) except Exception: logger.error(traceback.format_exc()) return run_state
def get_size(path, dirs_and_files=None): """ Get the size (in bytes) of the file or directory at or under the given path. Does not include symlinked files and directories. """ if parse_linked_bundle_url(path).uses_beam: return get_path_size(path) if os.path.islink(path) or not os.path.isdir(path): return os.lstat(path).st_size dirs_and_files = dirs_and_files or recursive_ls(path) return sum( os.lstat(path).st_size for path in itertools.chain(*dirs_and_files))
def check_disk_utilization(): running = True while running: start_time = time.time() try: disk_utilization = get_path_size(run_state.bundle_path) self.disk_utilization[run_state.bundle.uuid][ 'disk_utilization' ] = disk_utilization running = self.disk_utilization[run_state.bundle.uuid]['running'] except Exception: logger.error(traceback.format_exc()) end_time = time.time() # To ensure that we don't hammer the disk for this computation when # there are lots of files, we run it at most 10% of the time. time.sleep(max((end_time - start_time) * 10, 1.0))