def test_bytes2human_raises_expected_errors():
# Check TypeError(s)
for invalid in (None, '123', [1, 2, 3], 5 - 3j):
with pytest.raises(TypeError):
utils.bytes2human(invalid)
# Check ValueErrors(s)
for invalid in (-9.99, -7.5, -1, -0.4):
with pytest.raises(ValueError):
utils.bytes2human(invalid)
def test_cleanup_items_removes_and_yields_items_as_necessary(
self, monkeypatch, tmp_folder):
"""Test ``cleanup_items()`` if it successfully removes every item.
:param monkeypatch: pytest monkey patch fixture
:type monkeypatch: _pytest.monkeypatch.MonkeyPatch
:param tmp_folder: Test params and dummy test folder factory fixture pair.
:type tmp_folder: (dict, dirtools.tests.factory.DummyFolderFactory)
"""
params, _factory = tmp_folder
if params['total_items'] == 0:
return
# Use another factory folder that is not shared with other tests
with DummyFolderFactory(params['total_items'],
params['total_size'],
level=params['level']) as factory:
scan = Folder(factory.path,
params['sort_by'],
level=params['level'])
_humanise_item = Mock(side_effect=scan._humanise_item)
monkeypatch.setattr(scan, '_humanise_item', _humanise_item)
# cleanup must return a generator even the same (or greater) total size given
result = scan.cleanup_items(
bytes2human(scan.total_size, precision=11))
assert isinstance(result, types.GeneratorType)
with pytest.raises(StopIteration):
next(result)
_humanise_item.assert_not_called()
# Get a copy of items and remove one by one
items = tuple(scan.items(humanise=False))
_humanise_item.assert_not_called()
for item in items:
last_total = scan.total_size
last_items_len = scan._items_len
mines_one_byte = bytes2human(scan.total_size - 1, precision=11)
full_path = os.path.abspath(
os.path.join(factory.path, item['name']))
assert os.path.exists(full_path)
# attempt to remove single item
deleted = next(
scan.cleanup_items(mines_one_byte, humanise=False))
_humanise_item.assert_not_called()
assert deleted == item
assert scan.total_size == (last_total - deleted['size'])
assert scan._items_len == last_items_len - 1
assert not os.path.exists(full_path)
async def _scan(self, sort_by: SortBy) -> None:
"""Async process that has been initialised at the time of object instantiation.
It triggers bunch of other blocking / non-blocking methods and calculates the final :attr:`.exec_took`.
Public methods should use :meth:`_await` to wait for this internal scanning to be completed.
:param sort_by: SortBy enum attribute
:type sort_by: SortBy
:rtype: None
"""
logger.debug("Scanning initialised")
# those 2 lines are pretty much the BOTTLENECK of entire app.
async for item in self._iter_items(self._root):
self._insert_sorted(item, sort_by)
self.resort(sort_by)
self.exec_took = round(time.time() - self.exec_took, 3)
logger.debug(
"Scanning completed for {len:d} items with {size} of data; took {exec} second(s)."
.format(
len=self._items_len,
size=utils.bytes2human(self._total_size),
exec=self.exec_took,
))
def _humanise_item(cls, item: dict, precision: int) -> dict:
humanised = item.copy()
humanised["size"] = utils.bytes2human(item["size"],
precision=precision)
humanised["atime"] = time.strftime(cls._time_format,
time.gmtime(item["atime"]))
humanised["mtime"] = time.strftime(cls._time_format,
time.gmtime(item["mtime"]))
humanised["ctime"] = time.strftime(cls._time_format,
time.gmtime(item["ctime"]))
return humanised
def cleanup_items(self,
max_total_size: str,
humanise: bool = True,
precision: int = 2) -> Iterator[dict]:
"""Completely remove every item starting from the first in given sorting order until it reaches to ``max_total_size`` parameter.
Returns empty generator if the given ``max_total_size`` parameter is equal or greater than entire total size. Otherwise removes and yields every deleted item.
Blocks until the scanning operation has been completed on first access.
.. warning::
Result of this method is to DELETE the physical files / folders
on your disk until the given size matches the actual size
and there is NO UNDO for this operation.
:param max_total_size: Human representation of total desired size.
See: :func:``dirtools.utils.human2bytes``.
:type max_total_size: str
:param humanise: Humanise flag (required, no default value).
:type humanise: bool
:param precision: The floating precision of the human-readable size format (defaults to 2).
:type precision: int
:return: iterator
"""
self._await()
# Start deleting in the sorted order
old_len = self._items_len
old_size = self._total_size
max_total_size = utils.human2bytes(max_total_size)
while self._total_size > max_total_size:
item = self._items.popleft()
self._total_size -= item["size"]
self._items_len -= 1
item_path = os.path.abspath(os.path.join(self._root, item["name"]))
# REMOVE THE ITEM PERMANENTLY
try:
shutil.rmtree(item_path)
except NotADirectoryError:
os.remove(item_path)
# yield removed item
yield self._humanise_item(item, precision) if humanise else item
# Reduced to desired size
logger.debug(
"{del_len:d} items with total of {del_size} data has been deleted."
.format(
del_len=old_len - self._items_len,
del_size=utils.bytes2human(old_size - self._total_size),
))
def test_invoke_trimming_down_instead_of_listing(self, monkeypatch,
tmp_folder,
clone_factory):
"""
:param monkeypatch: pytest monkey patch fixture
:type monkeypatch: _pytest.monkeypatch.MonkeyPatch
:param tmp_folder: Test params and dummy test folder factory fixture pair.
:type tmp_folder: (dict, dirtools.tests.factory.DummyFolderFactory)
:param clone_factory: Factory to create `Folder` clone instance.
:type clone_factory: dirtools.tests.conftest.clone_factory._factory
:return:
"""
params, _factory = tmp_folder
if params['total_items'] == 0:
return
trim_down = int(human2bytes(params['total_size']) / 2)
trim_down_human = bytes2human(trim_down)
# Use another factory folder that is not shared with other tests
with DummyFolderFactory(params['total_items'],
params['total_size'],
level=params['level']) as factory:
# Create a scanner mock from the factory
scan = clone_factory(factory.path,
params['sort_by'],
level=params['level'])
FolderScanMock = Mock(return_value=scan)
monkeypatch.setattr(self.dirt, 'Folder', FolderScanMock)
# Give only numeric trim-down value that shouldn't be accepted
result = self.runner.invoke(self.dirt.invoke_dirtools3, [
factory.path, '-s',
str(params['sort_by']).lower(), '--trim-down',
str(trim_down)
])
assert result.exception is None
assert '--trim-down value cannot be only numeric' in result.output
result = self.runner.invoke(self.dirt.invoke_dirtools3, [
factory.path, '-s',
str(params['sort_by']).lower(), '--trim-down', trim_down_human
])
assert result.exception is None
scan.items.assert_not_called()
scan.cleanup_items.assert_called_once_with(
trim_down_human,
humanise=not self._DEFAULTS['nohuman'],
precision=self._DEFAULTS['precision'])
def test_each_item_has_been_scanned_correctly(self, monkeypatch,
tmp_folder):
"""Compare every scanned item to the testing factory class.
:param monkeypatch: pytest monkey patch fixture
:type monkeypatch: _pytest.monkeypatch.MonkeyPatch
:param tmp_folder: Test params and dummy test folder factory fixture pair.
:type tmp_folder: (dict, dirtools.tests.factory.DummyFolderFactory)
"""
params, factory = tmp_folder
scan = Folder(factory.path, params['sort_by'], params['level'])
_humanise_item = Mock(side_effect=scan._humanise_item)
monkeypatch.setattr(scan, '_humanise_item', _humanise_item)
factory_items = list(factory.items)
scanned_human = list(scan.items(humanise=True))
scanned_nonhuman = list(scan.items(humanise=False))
_humanise_item.assert_has_calls(
call(s, precision=2) for s in scanned_nonhuman)
_humanise_item.reset_mock()
assert len(factory_items) == len(scanned_nonhuman) == len(
scan) == scan._items_len
for item in factory_items:
raw = next(i for i in scanned_nonhuman
if i['name'] == item['name'])
human = next(i for i in scanned_human if i['name'] == item['name'])
assert tuple(raw.keys()) == tuple(human.keys()) == tuple(
item.keys())
assert raw['size'] == item['size']
assert human['size'] == bytes2human(item['size'])
assert raw['depth'] == item['depth']
assert raw['num_of_files'] == item['num_of_files']
assert raw['atime'] == item['atime']
assert raw['mtime'] == item['mtime']
def test_public_methods_should_block_before_processing(
self, monkeypatch, tmp_folder):
"""Make sure public methods and properties blocks until scanning
finishes.
:param monkeypatch: pytest monkey patch fixture
:type monkeypatch: _pytest.monkeypatch.MonkeyPatch
:param tmp_folder: Test params and dummy test folder factory fixture pair.
:type tmp_folder: (dict, dirtools.tests.factory.DummyFolderFactory)
"""
params, factory = tmp_folder
# Mock the _await internal function first
scan = Folder(factory.path, params['sort_by'], params['level'])
_ = scan.items()
blocker = Mock()
monkeypatch.setattr(scan, '_await', blocker)
# len(scan)
assert len(scan) == factory.total_items
blocker.assert_called_once()
blocker.reset_mock()
# .total_size
assert scan.total_size == factory.total_size
blocker.assert_called_once()
blocker.reset_mock()
# .items
_ = scan.items()
blocker.assert_called_once()
blocker.reset_mock()
# .cleanup_items() calls human2bytes once
_human2bytes = Mock(side_effect=human2bytes)
monkeypatch.setattr('dirtools.utils.human2bytes', _human2bytes)
factory_size_human = bytes2human(factory.total_size, precision=11)
with pytest.raises(StopIteration):
next(scan.cleanup_items(factory_size_human))
_human2bytes.assert_called_once_with(factory_size_human)
blocker.assert_called_once()
blocker.reset_mock()
# So give -1 to make it block actually, run only for >0 sizes
if factory.total_size > 0:
sh_rmtree = Mock()
os_remove = Mock()
monkeypatch.setattr('shutil.rmtree', sh_rmtree)
monkeypatch.setattr(os, 'remove', os_remove)
# Attempt to remove single item
next(
scan.cleanup_items(
bytes2human(factory.total_size - 1, precision=11)))
blocker.assert_called_once()
blocker.reset_mock()
assert sh_rmtree.called or os_remove.called
# .resort() called at the end of async loop, so it shouldn't _await
scan.resort(params['sort_by'])
blocker.assert_not_called()
blocker.reset_mock()
def test_bytes2human_calculates_correct_bytes_to_human():
for byte_val, human in VALID_BYTES2HUMAN:
# Also assert well-formatted human value to bytes
assert byte_val == utils.human2bytes(human)
assert utils.bytes2human(byte_val, precision=11) == human
def invoke_dirtools3(args):
"""Command line interface to the dirtools package."""
# Get SortBy enum val
try:
sortby = next(s for s in SortBy if args.sortby.upper() == str(s))
except StopIteration:
sys.stderr.write("Invalid sort by option: {0}".format(sortby))
return
path = args.path
precision = args.precision
depth = args.depth
nohuman = args.nohuman
trim_down = args.trim_down
output = args.output
# Create folder object and start its scanning process
scan = Folder(path, sortby, level=depth)
old_size = scan.total_size
old_items_len = len(scan)
# Only folder scanning and listing etc
if trim_down is None:
items = scan.items(humanise=not nohuman, precision=precision)
# Do not allow to pass only digit value because it will be interpreted as
# byte value and probably this was an accident.
elif trim_down.isdigit():
sys.stderr.write(
"--trim-down value cannot be only numeric to prevent accident, {0} given.".format(
trim_down
)
)
return
# Folder trimming
else:
items = scan.cleanup_items(trim_down, humanise=not nohuman, precision=precision)
# CSV - custom
if output.lower() == "csv":
with io.StringIO() as csv_io:
writer = csv.writer(csv_io, quoting=csv.QUOTE_NONNUMERIC)
writer.writerow(TABLE_HEADERS.values())
writer.writerows(i.values() for i in items)
rows = csv_io.getvalue().rstrip()
# Display it with tabulate
else:
rows = tabulate(items, TABLE_HEADERS, tablefmt=output, stralign="right")
sys.stdout.write(rows)
# Give summary info regarding to its listing
lit = lambda n, sing, plur: str(n) + (f" {sing}" if n == 1 else f" {plur}")
if trim_down is None:
sys.stdout.write(
"\n{ct} with total of {size} data; took {exec}.".format(
exec=lit(scan.exec_took, "second", "seconds"),
size=bytes2human(scan.total_size, precision=precision),
ct=lit(len(scan), "item", "items"),
)
)
# or cleaning operation
else:
del_len = len(scan) - old_items_len
del_size = bytes2human(old_size - scan.total_size, precision=precision)
sys.stderr.write(
"{del_len:d} items with total of {del_size} data has been deleted.".format(
del_len=del_len, del_size=del_size
)
)
sys.stderr.write(
"Currently {len} items left with {size} of data; took {exec} second(s).".format(
len=len(scan),
size=bytes2human(scan.total_size, precision=precision),
exec=scan.exec_took,
)
)