def test_find_duplicates_with_sort_key_modification_date(self):
items = [
temp_content('file', 'src/a/kiwi_03.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/b/kiwi_02.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/c/kiwi_01.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
]
def _ptf(test):
file_util.set_modification_date(f'{test.src_dir}/c/kiwi_01.jpg',
datetime.now())
file_util.set_modification_date(f'{test.src_dir}/b/kiwi_02.jpg',
datetime.now() - timedelta(days=1))
file_util.set_modification_date(f'{test.src_dir}/a/kiwi_03.jpg',
datetime.now() - timedelta(days=2))
t = self._call_find_duplicates(
extra_content_items=items,
recursive=True,
sort_key=file_duplicates_options.sort_key_modification_date,
pre_test_function=_ptf)
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/a/kiwi_03.jpg', [
f'{t.src_dir}/b/kiwi_02.jpg',
f'{t.src_dir}/c/kiwi_01.jpg',
]),
]), t.result.items)
def test_partition_with_media_type_and_threshold(self):
items = [
temp_content('file', 'src/apple.jpg',
unit_test_media.JPG_SMALLEST_POSSIBLE, 0o0644),
temp_content('file', 'src/kiwi.jpg',
unit_test_media.JPG_SMALLEST_POSSIBLE, 0o0644),
temp_content('file', 'src/barolo.mp4',
unit_test_media.MP4_SMALLEST_POSSIBLE, 0o0644),
temp_content('file', 'src/yogurt.foo', unit_test_media.UNKNOWN,
0o0644),
temp_content('file', 'src/zabaglione.foo', unit_test_media.UNKNOWN,
0o0644),
]
t = self._partition_test(extra_content_items=items,
dst_dir_same_as_src=False,
recursive=False,
partition_type='media_type',
threshold=2,
flatten=True)
dst_after_expected = [
'image',
'image/apple.jpg',
'image/kiwi.jpg',
]
self.assert_filename_list_equal(dst_after_expected, t.dst_files)
src_after_expected = [
'barolo.mp4',
'yogurt.foo',
'zabaglione.foo',
]
self.assert_filename_list_equal(src_after_expected, t.src_files)
def test_find_duplicates_delete_with_keep_empty_dirs(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
]
t = self._find_dups_test(extra_content_items=items,
recursive=True,
delete=True,
keep_empty_dirs=True)
self.assert_string_equal_fuzzy(
f'''\
{t.src_dir}/a/kiwi.jpg:
{t.src_dir}/b/kiwi_dup1.jpg
{t.src_dir}/c/kiwi_dup2.jpg
''', t.result.output)
src_after_expected = [
'a',
'a/apple.jpg',
'a/kiwi.jpg',
'a/lemon.jpg',
'b',
'c',
]
self.assert_filename_list_equal(src_after_expected, t.src_files)
def test_grep(self):
t = self._test_grep([
temp_content('file', 'fruit/lemon.py', f'{self._LEMON_PY}',
0o0644),
temp_content('file', 'fruit/kiwi.py', f'{self._KIWI_PY}', 0o0644),
temp_content('file', 'orange.py', f'{self._ORANGE_PY}', 0o0644),
],
'foo',
'function',
word_boundary=True)
self.assert_string_equal_fuzzy(
r'''
[
[
"fruit/kiwi.py",
" def foo(self):\n return 1",
[
[
3,
" def foo(self):"
],
[
4,
" return 1"
]
]
],
[
"fruit/lemon.py",
" def foo(self):\n return 1",
[
[
4,
" def foo(self):"
],
[
5,
" return 1"
]
]
],
[
"orange.py",
" def foo(self):\n return 1",
[
[
3,
" def foo(self):"
],
[
4,
" return 1"
]
]
]
]
''', t.json)
def test_find_duplicates_without_empty_files(self):
items = [
temp_content('file', 'src/a/empty.jpg', '', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/empty_dup1.jpg', '', 0o0644),
temp_content('file', 'src/c/empty_dup2.jpg', '', 0o0644),
]
t = self._call_find_duplicates(extra_content_items=items,
recursive=True,
include_empty_files=False)
self.assertEqual(self._xp_result_item_list([]), t.result.items)
def test_find_and_unsplit_dry_run(self):
items = [
temp_content('file', 'src/a/foo/kiwi.txt', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/parts/xfoo.txt.001', 'garbage',
0o0644),
temp_content('file', 'src/a/parts/foo.txt.001', 'foo001', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.002', 'foo002', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003', 'foo003', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.xx4', 'garbage', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003.garbage', 'garbage',
0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.01', 'lemon01',
0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.02', 'lemon02',
0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.03', 'lemon03',
0o0644),
]
t = self._find_and_unsplit_test(extra_content_items=items,
recursive=True,
dry_run=True)
self.assertEqual([
'a',
'a/foo',
'a/foo/kiwi.txt',
'a/parts',
'a/parts/foo.txt.001',
'a/parts/foo.txt.002',
'a/parts/foo.txt.003',
'a/parts/foo.txt.003.garbage',
'a/parts/foo.txt.xx4',
'a/parts/xfoo.txt.001',
'b',
'b/icons',
'b/icons/lemon.jpg.01',
'b/icons/lemon.jpg.02',
'b/icons/lemon.jpg.03',
], t.src_files)
self.assert_string_equal_fuzzy(
f'''\
{t.src_dir}/a/parts/foo.txt:
{t.src_dir}/a/parts/foo.txt.001
{t.src_dir}/a/parts/foo.txt.002
{t.src_dir}/a/parts/foo.txt.003
{t.src_dir}/b/icons/lemon.jpg:
{t.src_dir}/b/icons/lemon.jpg.01
{t.src_dir}/b/icons/lemon.jpg.02
{t.src_dir}/b/icons/lemon.jpg.03
''', t.result.output)
return
def test_find_file_duplicates_with_setup_and_removed_resolved_file(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/brie.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/cheddar.jpg', 'this is cheddar',
0o0644),
temp_content('file', 'foo/cheese/gouda.jpg', 'this is lemon',
0o0644),
]
options = file_duplicates_options(recursive=True)
with dir_operation_tester(extra_content_items=items) as t:
setup = file_duplicates.setup([t.src_dir], options=options)
file_util.remove(f'{t.src_dir}/a/kiwi.jpg')
dups = file_duplicates.find_file_duplicates_with_setup(
f'{t.tmp_dir}/foo/cheese/brie.jpg', setup)
self.assertEqual([
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
], dups)
dups = file_duplicates.find_file_duplicates_with_setup(
f'{t.tmp_dir}/foo/cheese/gouda.jpg', setup)
self.assertEqual([
f'{t.src_dir}/a/lemon.jpg',
], dups)
def test_find_and_unsplit_existing_target_same(self):
items = [
temp_content('file', 'src/a/foo/kiwi.txt', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/parts/foo.txt', 'foo001foo002foo003', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.001', 'foo001', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.002', 'foo002', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003', 'foo003', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.01', 'lemon01', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.02', 'lemon02', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.03', 'lemon03', 0o0644),
]
t = self._find_and_unsplit_test(extra_content_items = items,
recursive = True)
self.assertEqual( [
'a',
'a/foo',
'a/foo/kiwi.txt',
'a/parts',
'a/parts/foo.txt',
'b',
'b/icons',
'b/icons/lemon.jpg',
], t.src_files )
self.assert_text_file_equal( 'foo001foo002foo003', f'{t.src_dir}/a/parts/foo.txt' )
self.assert_text_file_equal( 'lemon01lemon02lemon03', f'{t.src_dir}/b/icons/lemon.jpg' )
def test_find_and_unsplit_existing_target_different(self):
items = [
temp_content('file', 'src/a/foo/kiwi.txt', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/parts/foo.txt', 'different', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.001', 'foo001', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.002', 'foo002', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003', 'foo003', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.01', 'lemon01', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.02', 'lemon02', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.03', 'lemon03', 0o0644),
]
ts = datetime(year = 2000, month = 1, day = 1, hour = 1, second = 1)
t = self._find_and_unsplit_test(extra_content_items = items,
recursive = True,
existing_file_timestamp = ts)
self.assertEqual( [
'a',
'a/foo',
'a/foo/kiwi.txt',
'a/parts',
'a/parts/foo-20000101010001.txt',
'a/parts/foo.txt',
'b',
'b/icons',
'b/icons/lemon.jpg',
], t.src_files )
self.assert_text_file_equal( 'foo001foo002foo003', f'{t.src_dir}/a/parts/foo-20000101010001.txt' )
self.assert_text_file_equal( 'different', f'{t.src_dir}/a/parts/foo.txt' )
self.assert_text_file_equal( 'lemon01lemon02lemon03', f'{t.src_dir}/b/icons/lemon.jpg' )
def test_find_and_unsplit_with_check_downloading(self):
items = [
temp_content('file', 'src/a/foo/kiwi.txt', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.001', 'part001', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.002', 'part002', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003', 'part003', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003.part', 'foo', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.01', 'part01', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.02', 'part02', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.03', 'part03', 0o0644),
]
t = self._find_and_unsplit_test(extra_content_items = items,
recursive = True,
check_downloading = True)
self.assertEqual( [
'a',
'a/foo',
'a/foo/kiwi.txt',
'a/parts',
'a/parts/foo.txt.001',
'a/parts/foo.txt.002',
'a/parts/foo.txt.003',
'a/parts/foo.txt.003.part',
'b',
'b/icons',
'b/icons/lemon.jpg',
], t.src_files )
def test_rename_dirs_wont_leak_above_root_dir(self):
'Test that we only rename dirs starting at root_dir'
tmp_dir = self._make_temp_content([
temp_content('file', 'fruit/fruit/lib/fruit/kiwi.py', 'this is kiwi', 0o0644),
temp_content('file', 'fruit/fruit/lib/fruit/lemon.py', 'this is lemon', 0o0644),
temp_content('file', 'fruit/fruit/lib/fruit/constants.py', 'this is constants', 0o0644),
temp_content('file', 'fruit/fruit/lib/fruit/constants2.py', 'this is constants2', 0o0644),
temp_content('file', 'fruit/fruit/lib/fruity/constants2b.py', 'this is constants2', 0o0644),
temp_content('file', 'fruit/fruit/tests/lib/fruit/test_kiwi.py', 'this is test kiwi', 0o0644),
temp_content('file', 'fruit/fruit/tests/lib/fruit/test_lemon.py', 'this is test lemon', 0o0644),
temp_content('file', 'fruit/fruit/tests/lib/fruity/test_lemonb.py', 'this is test lemon', 0o0644),
])
options = refactor_options(word_boundary = False)
refactor_files.rename_dirs(path.join(tmp_dir, 'fruit'), 'fruit', 'cheese', options = options)
self.assert_filename_list_equal( [
'fruit',
'fruit/cheese',
'fruit/cheese/lib',
'fruit/cheese/lib/cheese',
'fruit/cheese/lib/cheese/constants.py',
'fruit/cheese/lib/cheese/constants2.py',
'fruit/cheese/lib/cheese/kiwi.py',
'fruit/cheese/lib/cheese/lemon.py',
'fruit/cheese/lib/cheesey',
'fruit/cheese/lib/cheesey/constants2b.py',
'fruit/cheese/tests',
'fruit/cheese/tests/lib',
'fruit/cheese/tests/lib/cheese',
'fruit/cheese/tests/lib/cheese/test_kiwi.py',
'fruit/cheese/tests/lib/cheese/test_lemon.py',
'fruit/cheese/tests/lib/cheesey',
'fruit/cheese/tests/lib/cheesey/test_lemonb.py',
], file_find.find(tmp_dir, file_type = file_find.ANY) )
def test_find_duplicates(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
]
t = self._find_dups_test(extra_content_items=items, recursive=True)
self.assert_string_equal_fuzzy(
f'''\
{t.src_dir}/a/kiwi.jpg:
{t.src_dir}/b/kiwi_dup1.jpg
{t.src_dir}/c/kiwi_dup2.jpg
''', t.result.output)
def test_threshold(self):
extra_content_items = [
temp_content('file', 'src/apple.txt', 'this is apple.txt', 0o0644),
temp_content('file', 'src/kiwi.txt', 'this is kiwi.txt', 0o0644),
temp_content('file', 'src/lemon.txt', 'this is lemon.txt', 0o0644),
temp_content('file', 'src/blueberry.txt', 'this is blueberry.txt',
0o0644),
]
t = self._split_test(extra_content_items=extra_content_items,
threshold=5)
expected = [
'apple.txt',
'blueberry.txt',
'kiwi.txt',
'lemon.txt',
]
self.assert_filename_list_equal([], t.dst_files)
self.assert_filename_list_equal(expected, t.src_files)
def test_partition_with_prefix_dry_run(self):
items = [
temp_content('file', 'src/readme.md', 'readme.md', 0o0644),
temp_content('file', 'src/a/kiwi-10.jpg', 'kiwi-10.txt', 0o0644),
temp_content('file', 'src/a/kiwi-20.jpg', 'kiwi-20.txt', 0o0644),
temp_content('file', 'src/a/kiwi-30.jpg', 'kiwi-30.txt', 0o0644),
temp_content('file', 'src/b/lemon-10.jpg', 'lemon-10.txt', 0o0644),
temp_content('file', 'src/b/lemon-20.jpg', 'lemon-20.txt', 0o0644),
temp_content('file', 'src/b/lemon-30.jpg', 'lemon-30.txt', 0o0644),
temp_content('file', 'src/c/cheese-10.jpg', 'cheese-10.jpg', 0o0644),
temp_content('file', 'src/icons/foo.png', 'foo.png', 0o0644),
]
t = self._partition_test(extra_content_items = items,
dst_dir_same_as_src = False,
recursive = True,
partition_type = 'prefix',
dry_run = True,
delete_empty_dirs = True,
flatten = True)
dst_after_expected = [
]
self.assert_filename_list_equal( dst_after_expected, t.dst_files )
src_after_expected = [
'a',
'a/kiwi-10.jpg',
'a/kiwi-20.jpg',
'a/kiwi-30.jpg',
'b',
'b/lemon-10.jpg',
'b/lemon-20.jpg',
'b/lemon-30.jpg',
'c',
'c/cheese-10.jpg',
'icons',
'icons/foo.png',
'readme.md',
]
self.assert_filename_list_equal( src_after_expected, t.src_files )
self.assert_string_equal_fuzzy(f'''\
{t.src_dir}/a/kiwi-10.jpg => {t.dst_dir}/kiwi/kiwi-10.jpg
{t.src_dir}/a/kiwi-20.jpg => {t.dst_dir}/kiwi/kiwi-20.jpg
{t.src_dir}/a/kiwi-30.jpg => {t.dst_dir}/kiwi/kiwi-30.jpg
{t.src_dir}/b/lemon-10.jpg => {t.dst_dir}/lemon/lemon-10.jpg
{t.src_dir}/b/lemon-20.jpg => {t.dst_dir}/lemon/lemon-20.jpg
{t.src_dir}/b/lemon-30.jpg => {t.dst_dir}/lemon/lemon-30.jpg
{t.src_dir}/c/cheese-10.jpg => {t.dst_dir}/cheese/cheese-10.jpg
''', t.result.output )
def test_partition_with_two_files_to_partition(self):
items = [
temp_content('file', 'src/a/kiwi-10.jpg', 'kiwi-10.txt', 0o0644),
temp_content('file', 'src/b/kiwi-20.jpg', 'kiwi-20.txt', 0o0644),
]
t = self._partition_test(extra_content_items=items,
dst_dir_same_as_src=False,
recursive=True,
partition_type='prefix',
delete_empty_dirs=True,
flatten=True)
dst_after_expected = [
'kiwi',
'kiwi/kiwi-10.jpg',
'kiwi/kiwi-20.jpg',
]
self.assert_filename_list_equal(dst_after_expected, t.dst_files)
src_after_expected = []
self.assert_filename_list_equal(src_after_expected, t.src_files)
def test_find_duplicates_correct_order(self):
items = [
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/z/kiwi.jpg', 'this is kiwi', 0o0644),
]
t = self._call_find_duplicates(extra_content_items=items,
recursive=True)
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/b/kiwi_dup1.jpg', [
f'{t.src_dir}/c/kiwi_dup2.jpg',
f'{t.src_dir}/z/kiwi.jpg',
]),
]), t.result.items)
def test_rename_dirs_one_file(self):
tmp_dir = self._make_temp_content([
temp_content('file', 'kiwi/xdata2/kiwi_stuff2/kiwi2.txt', 'foo.txt', 0o0644),
])
options = refactor_options(word_boundary = False)
refactor_files.rename_dirs(tmp_dir, 'kiwi', 'chocolate', options = options)
self.assert_filename_list_equal( [
'chocolate',
'chocolate/xdata2',
'chocolate/xdata2/chocolate_stuff2',
'chocolate/xdata2/chocolate_stuff2/kiwi2.txt',
], file_find.find(tmp_dir, file_type = file_find.ANY) )
def test_find_and_unsplit_with_unzip_and_multiple_members(self):
tmp = temp_archive.make_temp_archive([
temp_archive.item('kiwi.mp4', filename = self.mp4_file),
temp_archive.item('lemon.jpg', filename = self.jpg_file),
], 'zip')
tmp_dir = self.make_temp_dir()
files = file_split.split_file(tmp, int(file_util.size(tmp) / 3),
output_directory = tmp_dir,
zfill_length = 3)
self.assertEqual( 4, len(files) )
items = [
temp_content('file', 'src/kiwi.mp4.zip.001', file_util.read(files[0]), 0o0644),
temp_content('file', 'src/kiwi.mp4.zip.002', file_util.read(files[1]), 0o0644),
temp_content('file', 'src/kiwi.mp4.zip.003', file_util.read(files[2]), 0o0644),
temp_content('file', 'src/kiwi.mp4.zip.004', file_util.read(files[3]), 0o0644),
]
t = self._find_and_unsplit_test(extra_content_items = items,
unzip = True)
self.assertEqual( [
'kiwi.mp4.zip',
], t.src_files )
def test_find_duplicates_with_sort_key_basename_length(self):
items = [
temp_content('file', 'src/a/kiwi_12345.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_1234.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_123.jpg', 'this is kiwi', 0o0644),
]
t = self._call_find_duplicates(
extra_content_items=items,
recursive=True,
sort_key=file_duplicates_options.sort_key_basename_length)
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/c/kiwi_123.jpg', [
f'{t.src_dir}/b/kiwi_1234.jpg',
f'{t.src_dir}/a/kiwi_12345.jpg',
]),
]), t.result.items)
def test_find_duplicates_with_sort_key(self):
items = [
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/z/kiwi.jpg', 'this is kiwi', 0o0644),
]
sort_key = lambda filename: 0 if 'z' in file_path.split(filename
) else 1
t = self._call_find_duplicates(extra_content_items=items,
recursive=True,
sort_key=sort_key)
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/z/kiwi.jpg', [
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
]),
]), t.result.items)
def test_find_duplicates_with_setup(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
]
options = file_duplicates_options(recursive=True)
with dir_operation_tester(extra_content_items=items) as t:
setup = file_duplicates.setup([t.src_dir], options=options)
t.result = file_duplicates.find_duplicates_with_setup(setup)
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/a/kiwi.jpg', [
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
]),
]), t.result.items)
def test_resolve_python_files(self):
tmp_dir = self._make_temp_content([
temp_content('file', 'fruit/icons/kiwi.txt', 'foo.txt', 0o0644),
temp_content('file', 'fruit/icons/berry_wrong.py', b'\x00\xde\xad\xbe\xef\x00', 0o0644),
temp_content('file', 'fruit/src/lemon.py', "class barolo(object): pass", 0o0644),
temp_content('file', 'fruit/bin/fscript', "#!/usr/bin/env python3\na=666\n", 0o0755),
temp_content('file', 'wine/icons/chablis.txt', 'foo.txt', 0o0644),
temp_content('file', 'wine/icons/sherry_wrong.py', b'\x00\xde\xad\xbe\xef\x00', 0o0644),
temp_content('file', 'wine/src/barolo.py', "class barolo(object): pass", 0o0644),
temp_content('file', 'wine/bin/wscript', "#!/usr/bin/env python3\na=666\n", 0o0755),
temp_content('file', 'foo.py', "class foo(object): pass", 0o0644),
])
self.assert_filename_list_equal( [
f'{tmp_dir}/foo.py',
f'{tmp_dir}/fruit/bin/fscript',
f'{tmp_dir}/fruit/src/lemon.py',
f'{tmp_dir}/wine/bin/wscript',
f'{tmp_dir}/wine/src/barolo.py',
], refactor_files.resolve_python_files([ path.join(tmp_dir, f) for f in [ 'fruit', 'wine', 'foo.py' ] ]).to_list() )
def test_one_file(self):
extra_content_items = [
temp_content('file', 'src/foo.txt', b'this is foo.txt', 0o0644),
]
t = self._split_test([],
1,
1,
extra_content_items=extra_content_items,
recursive=True)
expected = [
'chunk-1',
'chunk-1/foo.txt',
]
self.assert_filename_list_equal(expected, t.dst_files)
self.assert_filename_list_equal([], t.src_files)
def test_nothing_to_split(self):
extra_content_items = [
temp_content('file', 'src/foo.txt', b'this is foo.txt', 0o0644),
]
def _ptf(test):
file_util.remove(f'{test.src_dir}/foo.txt')
t = self._split_test([],
1,
1,
extra_content_items=extra_content_items,
pre_test_function=_ptf)
expected = []
self.assert_filename_list_equal(expected, t.dst_files)
self.assert_filename_list_equal([], t.src_files)
def test_find_and_unsplit_incomplete_set(self):
items = [
temp_content('file', 'src/a/foo/kiwi.txt', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.001', 'part001', 0o0644),
temp_content('file', 'src/a/parts/foo.txt.003', 'part003', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.01', 'part01', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.02', 'part02', 0o0644),
temp_content('file', 'src/b/icons/lemon.jpg.03', 'part03', 0o0644),
]
with self.assertRaises(file_split_error) as ctx:
self._find_and_unsplit_test(extra_content_items = items,
recursive = True)
def test_partition_with_prefix(self):
items = [
temp_content('file', 'src/readme.md', 'readme.md', 0o0644),
temp_content('file', 'src/a/kiwi-10.jpg', 'kiwi-10.txt', 0o0644),
temp_content('file', 'src/a/kiwi-20.jpg', 'kiwi-20.txt', 0o0644),
temp_content('file', 'src/a/kiwi-30.jpg', 'kiwi-30.txt', 0o0644),
temp_content('file', 'src/b/lemon-10.jpg', 'lemon-10.txt', 0o0644),
temp_content('file', 'src/b/lemon-20.jpg', 'lemon-20.txt', 0o0644),
temp_content('file', 'src/b/lemon-30.jpg', 'lemon-30.txt', 0o0644),
temp_content('file', 'src/c/cheese-10.jpg', 'cheese-10.jpg', 0o0644),
temp_content('file', 'src/icons/foo.png', 'foo.png', 0o0644),
]
t = self._partition_test(extra_content_items = items,
dst_dir_same_as_src = False,
recursive = True,
partition_type = 'prefix',
delete_empty_dirs = True,
flatten = True)
dst_after_expected = [
'cheese',
'cheese/cheese-10.jpg',
'kiwi',
'kiwi/kiwi-10.jpg',
'kiwi/kiwi-20.jpg',
'kiwi/kiwi-30.jpg',
'lemon',
'lemon/lemon-10.jpg',
'lemon/lemon-20.jpg',
'lemon/lemon-30.jpg',
]
self.assert_filename_list_equal( dst_after_expected, t.dst_files )
src_after_expected = [
'icons',
'icons/foo.png',
'readme.md',
]
self.assert_filename_list_equal( src_after_expected, t.src_files )
def test_match_files_with_word_boundary(self):
tmp_dir = self._make_temp_content([
temp_content('file', 'lib/fruit/kiwi.py', 'this is kiwi', 0o0644),
temp_content('file', 'lib/fruit/lemon.py', 'this is lemon', 0o0644),
temp_content('file', 'lib/fruit/constants.py', 'this is constants', 0o0644),
temp_content('file', 'lib/fruit/constants2.py', 'this is constants2', 0o0644),
temp_content('file', 'tests/lib/fruit/test_kiwi.py', 'this is test kiwi', 0o0644),
temp_content('file', 'tests/lib/fruit/test_lemon.py', 'this is test lemon', 0o0644),
])
options = refactor_options(word_boundary = True)
self.assert_filename_list_equal( [
f'{tmp_dir}/lib/fruit/kiwi.py',
f'{tmp_dir}/tests/lib/fruit/test_kiwi.py',
], refactor_files.match_files( file_find.find(tmp_dir, relative = False), 'kiwi', options = options) )
def test_find_file_duplicates_no_dups(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/brie.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/cheddar.jpg', 'this is cheddar',
0o0644),
]
t = self._call_find_file_duplicates('foo/cheese/cheddar.jpg',
extra_content_items=items,
recursive=True)
self.assertEqual([], t.result)
def test_find_duplicates_with_ignore_files(self):
ignore_file = self.make_temp_file(content=r'''
*.foo
''')
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/d/ignore.foo', 'this is kiwi', 0o0644),
]
t = self._call_find_duplicates(extra_content_items=items,
recursive=True,
ignore_files=[ignore_file])
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/a/kiwi.jpg', [
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
]),
]), t.result.items)
def test_function_add_arg(self):
tmp_dir = self._make_temp_content([
temp_content('file', 'fruit/lemon.py', f'{self._LEMON_PY}',
0o0644),
temp_content('file', 'fruit/kiwi.py', f'{self._KIWI_PY}', 0o0644),
temp_content('file', 'orange.py', f'{self._ORANGE_PY}', 0o0644),
])
options = refactor_options(word_boundary=True)
refactor_ast.function_add_arg([tmp_dir],
'foo',
'added_arg',
options=options)
self.assert_text_file_equal_fuzzy(
'''\
from .kiwi import kiwi
class lemon(kiwi):
def foo(self, added_arg):
return 1
def bar(self):
return 2
''', f'{tmp_dir}/fruit/lemon.py')
self.assert_text_file_equal_fuzzy(
'''\
class kiwi(object):
def foo(self, added_arg):
return 1
def bar(self):
return 2
''', f'{tmp_dir}/fruit/kiwi.py')
self.assert_text_file_equal_fuzzy(
'''\
class orange(kiwi):
def foo(self, added_arg):
return 1
def bar(self):
return 2
def foo2(self):
return 3
def foo_prime(self):
return 3
def foo_test(self, *kargs, **kwargs):
return 3
def foo_test2(self, a, b, sweet = True):
return 3
def foo_test3(self, *kargs):
return 3
def foo_test4(self, **kwargs):
return 3
def foo_test5(self, a, b,
c, d):
return 3
''', f'{tmp_dir}/orange.py')
