def test_size():
with temporary_dir() as td:
create_files(td, 'file1.txt')
file1 = os.path.join(td, 'file1.txt')
assert safe_size(file1) == 0
with open(file1, 'w') as fp:
fp.write('!' * 101)
assert safe_size(file1) == 101
f1stat = os.stat(file1)
assert safe_bsize(file1) == 512 * f1stat.st_blocks
assert du(td) == safe_bsize(file1)
file2 = os.path.join(td, 'file2.txt')
os.symlink('file1.txt', file2)
assert safe_size(file2) == len('file1.txt')
assert safe_bsize(file2) == len('file1.txt')
assert du(td) == safe_bsize(file1) + len('file1.txt')
assert safe_size(os.path.join(td, 'file3.txt')) == 0
assert safe_bsize(os.path.join(td, 'file3.txt')) == 0
errors = []
def on_error(path, err):
errors.append(path)
safe_size(os.path.join(td, 'file3.txt'), on_error=on_error)
assert errors == [os.path.join(td, 'file3.txt')]
def test_size():
with temporary_dir() as td:
create_files(td, 'file1.txt')
file1 = os.path.join(td, 'file1.txt')
assert safe_size(file1) == 0
with open(file1, 'w') as fp:
fp.write('!' * 101)
assert safe_size(file1) == 101
f1stat = os.stat(file1)
assert safe_bsize(file1) == 512 * f1stat.st_blocks
assert du(td) == safe_bsize(file1)
file2 = os.path.join(td, 'file2.txt')
os.symlink('file1.txt', file2)
assert safe_size(file2) == len('file1.txt')
assert safe_bsize(file2) == len('file1.txt')
assert du(td) == safe_bsize(file1) + len('file1.txt')
assert safe_size(os.path.join(td, 'file3.txt')) == 0
assert safe_bsize(os.path.join(td, 'file3.txt')) == 0
errors = []
def on_error(path, err):
errors.append(path)
safe_size(os.path.join(td, 'file3.txt'), on_error=on_error)
assert errors == [os.path.join(td, 'file3.txt')]
def get_logs(self, task_id, with_size=True):
state = self.state(task_id)
if state and state.header:
for path in self._detector.get_process_logs(task_id, state.header.log_dir):
if with_size:
yield path, safe_bsize(path)
else:
yield path
def get_logs(self, with_size=True):
if self._state and self._state.header and self._state.header.log_dir:
for path in self._detector.get_process_logs(
self._task_id, self._state.header.log_dir):
if with_size:
yield path, safe_bsize(path)
else:
yield path
def get_logs(self, task_id, with_size=True):
state = self.state(task_id)
if state and state.header:
for path in self._detector.get_process_logs(
task_id, state.header.log_dir):
if with_size:
yield path, safe_bsize(path)
else:
yield path
def get_metadata(self, task_id, with_size=True):
runner_ckpt = self._detector.get_checkpoint(task_id)
process_ckpts = [ckpt for ckpt in self._detector.get_process_checkpoints(task_id)]
json_spec = TaskPath(root=self._root, task_id=task_id, state='finished').getpath('task_path')
for path in [json_spec, runner_ckpt] + process_ckpts:
if with_size:
yield path, safe_bsize(path)
else:
yield path
def get_data(self, with_size=True):
if self._state and self._state.header and self._state.header.sandbox:
for root, dirs, files in os.walk(self._state.header.sandbox):
for file in files:
filename = os.path.join(root, file)
if with_size:
yield filename, safe_bsize(filename)
else:
yield filename
def _initialize(self):
""" Collect an initial snapshot of the disk usage in the path """
log.debug("Starting watchdog observer to collect events...")
self._observer.schedule(self, path=self._path, recursive=True)
self._observer.start()
log.debug("Collecting initial disk usage sample...")
for root, _, files in os.walk(self._path):
for filename in files:
f = os.path.join(root, filename)
self._files[f] = safe_bsize(f)
def get_data(self, task_id, with_size=True):
state = self.state(task_id)
if state and state.header and state.header.sandbox:
for root, dirs, files in os.walk(state.header.sandbox):
for file in files:
filename = os.path.join(root, file)
if with_size:
yield filename, safe_bsize(filename)
else:
yield filename
def _initialize(self):
""" Collect an initial snapshot of the disk usage in the path """
log.debug("Starting watchdog observer to collect events...")
self._observer.schedule(self, path=self._path, recursive=True)
self._observer.start()
log.debug("Collecting initial disk usage sample...")
for root, _, files in os.walk(self._path):
for filename in files:
f = os.path.join(root, filename)
self._files[f] = safe_bsize(f)
def get_metadata(self, task_id, with_size=True):
runner_ckpt = self._detector.get_checkpoint(task_id)
process_ckpts = [
ckpt for ckpt in self._detector.get_process_checkpoints(task_id)
]
json_spec = TaskPath(root=self._root,
task_id=task_id,
state='finished').getpath('task_path')
for path in [json_spec, runner_ckpt] + process_ckpts:
if with_size:
yield path, safe_bsize(path)
else:
yield path
def get_metadata(self, with_size=True):
runner_ckpt = self._detector.get_checkpoint(self._task_id)
process_ckpts = [
ckpt
for ckpt in self._detector.get_process_checkpoints(self._task_id)
]
# assumes task is in finished state.
json_spec = self._pathspec.given(state='finished').getpath('task_path')
for path in [json_spec, runner_ckpt] + process_ckpts:
if with_size:
yield path, safe_bsize(path)
else:
yield path
def stat_file(path): self._files[path] = safe_bsize(path)
def stat_file(path):
self._files[path] = safe_bsize(path)
