def _move_files_test(self,
extra_content_items=None,
recursive=False,
dry_run=False,
dup_file_timestamp=None,
dup_file_count=None):
with dir_operation_tester(
extra_content_items=extra_content_items) as test:
file_util.mkdir(test.dst_dir)
args = [
'files',
'move',
test.src_dir,
test.dst_dir,
]
if recursive:
args.append('--recursive')
if dry_run:
args.append('--dry-run')
if dup_file_timestamp:
args.extend(['--dup-file-timestamp', dup_file_timestamp])
if dup_file_count:
args.extend(['--dup-file-count', str(dup_file_count)])
test.result = self.run_program(self._program, args)
return test
def _partition_test(
self,
extra_content_items=None,
dst_dir_same_as_src=False,
recursive=False,
files=None,
partition_type=dir_partition_defaults.PARTITION_TYPE,
partition_criteria=None,
pre_test_function=None,
delete_empty_dirs=dir_partition_defaults.DELETE_EMPTY_DIRS,
threshold=dir_partition_defaults.THRESHOLD,
flatten=dir_partition_defaults.FLATTEN):
with dir_operation_tester(
extra_content_items=extra_content_items,
dst_dir_same_as_src=dst_dir_same_as_src) as test:
options = dir_partition_options(
recursive=recursive,
dup_file_timestamp='dup-timestamp',
partition_type=partition_type,
partition_criteria=partition_criteria,
delete_empty_dirs=delete_empty_dirs,
threshold=threshold,
flatten=flatten,
dst_dir=test.dst_dir)
if pre_test_function:
pre_test_function(test)
if files:
files = [path.join(test.src_dir, f) for f in files]
else:
files = [test.src_dir]
test.result = dir_partition.partition(files, options=options)
return test
def _find_and_unsplit_test(
self,
extra_content_items=None,
recursive=False,
check_downloading=_DEFAULT_FILE_SPLIT_OPTIONS.check_downloading,
check_modified=_DEFAULT_FILE_SPLIT_OPTIONS.check_modified,
check_modified_interval=_DEFAULT_FILE_SPLIT_OPTIONS.
check_modified_interval,
dry_run=False):
options = file_split_options(recursive=recursive,
check_downloading=check_downloading)
with dir_operation_tester(
extra_content_items=extra_content_items) as test:
args = [
'file_split',
'unsplit',
test.src_dir,
]
if recursive:
args.append('--recursive')
if check_downloading:
args.append('--check-downloading')
if dry_run:
args.append('--dry-run')
test.result = self.run_program(self._program, args)
return test
def test_find_file_duplicates_with_setup_and_removed_resolved_file(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/brie.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'foo/cheese/cheddar.jpg', 'this is cheddar',
0o0644),
temp_content('file', 'foo/cheese/gouda.jpg', 'this is lemon',
0o0644),
]
options = file_duplicates_options(recursive=True)
with dir_operation_tester(extra_content_items=items) as t:
setup = file_duplicates.setup([t.src_dir], options=options)
file_util.remove(f'{t.src_dir}/a/kiwi.jpg')
dups = file_duplicates.find_file_duplicates_with_setup(
f'{t.tmp_dir}/foo/cheese/brie.jpg', setup)
self.assertEqual([
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
], dups)
dups = file_duplicates.find_file_duplicates_with_setup(
f'{t.tmp_dir}/foo/cheese/gouda.jpg', setup)
self.assertEqual([
f'{t.src_dir}/a/lemon.jpg',
], dups)
def _call_find_file_duplicates(self,
filename,
extra_content_items=None,
recursive=False,
small_checksum_size=1024 * 1024,
prefer_prefixes=None,
sort_key=None,
pre_test_function=None,
include_empty_files=False,
ignore_files=[]):
options = file_duplicates_options(
recursive=recursive,
small_checksum_size=small_checksum_size,
sort_key=sort_key,
include_empty_files=include_empty_files,
ignore_files=ignore_files)
with dir_operation_tester(
extra_content_items=extra_content_items) as test:
if pre_test_function:
pre_test_function(test)
if prefer_prefixes:
xglobals = {'test': test}
prefer_prefixes = [eval(x, xglobals) for x in prefer_prefixes]
options.prefer_prefixes = prefer_prefixes
test.result = file_duplicates.find_file_duplicates(path.join(
test.tmp_dir, filename), [test.src_dir],
options=options)
return test
def _split_test(self,
multiplied_content_items=None,
content_multiplier=1,
chunk_size=250,
extra_content_items=None,
dst_dir_same_as_src=False,
recursive=False,
sort_order='filename',
sort_reverse=False,
pre_test_function=None,
threshold=None):
options = dir_split_options(chunk_size=chunk_size,
prefix='chunk-',
recursive=recursive,
dup_file_timestamp='dup-timestamp',
sort_order=sort_order,
sort_reverse=sort_reverse,
threshold=threshold)
with dir_operation_tester(
multiplied_content_items=multiplied_content_items,
content_multiplier=content_multiplier,
extra_content_items=extra_content_items,
dst_dir_same_as_src=dst_dir_same_as_src) as test:
if pre_test_function:
pre_test_function(test)
dir_split.split(test.src_dir, test.dst_dir, options)
return test
def _test(self,
items,
files=[],
recursive=False,
dry_run=False,
flatten=dir_combine_defaults.FLATTEN,
delete_empty_dirs=dir_combine_defaults.DELETE_EMPTY_DIRS):
with dir_operation_tester(extra_content_items=items) as test:
if files:
files_args = [path.join(test.src_dir, f) for f in files]
else:
files_args = [test.src_dir]
args = [
'dir_combine',
'combine',
'--dup-file-timestamp',
'dup-timestamp',
'--dup-file-count',
'42',
] + files_args
if recursive:
args.append('--recursive')
if dry_run:
args.append('--dry-run')
if flatten:
args.append('--flatten')
if delete_empty_dirs:
args.append('--delete-empty-dirs')
test.result = self.run_program(self._program, args)
return test
def _partition_test(self,
extra_content_items = None,
dst_dir_same_as_src = False,
recursive = False,
partition_type = dir_partition_defaults.PARTITION_TYPE,
dry_run = False,
delete_empty_dirs = dir_partition_defaults.DELETE_EMPTY_DIRS,
threshold = dir_partition_defaults.THRESHOLD,
flatten = dir_partition_defaults.FLATTEN):
with dir_operation_tester(extra_content_items = extra_content_items,
dst_dir_same_as_src = dst_dir_same_as_src) as test:
args = [
'dir_partition',
'partition',
'--type', partition_type,
'--dst-dir', test.dst_dir,
test.src_dir,
]
if recursive:
args.append('--recursive')
if dry_run:
args.append('--dry-run')
if delete_empty_dirs:
args.append('--delete-empty-dirs')
if threshold:
args.extend([ '--threshold', str(threshold) ])
if flatten:
args.append('--flatten')
test.result = self.run_program(self._program, args)
return test
def _test(self, items, recursive=False, dry_run=False):
with dir_operation_tester(extra_content_items=items) as test:
args = [
'dirs',
'remove_empty',
test.src_dir,
]
if recursive:
args.append('--recursive')
if dry_run:
args.append('--dry-run')
test.result = self.run_program(self._program, args)
return test
def _split_test(self, multiplied_content_items, content_multiplier, chunk_size, extra_content_items = None):
with dir_operation_tester(multiplied_content_items = multiplied_content_items,
content_multiplier = content_multiplier,
extra_content_items = extra_content_items) as test:
args = [
'dir_split',
'split',
'--prefix', 'chunk-',
'--dst-dir', test.dst_dir,
'--chunk-size', str(chunk_size),
test.src_dir,
]
test.result = self.run_program(self._program, args)
return test
def _combine_test(self,
extra_content_items = None,
dst_dir_same_as_src = False,
recursive = False,
files = None,
flatten = dir_combine_defaults.FLATTEN,
delete_empty_dirs = dir_combine_defaults.DELETE_EMPTY_DIRS):
options = dir_combine_options(recursive = recursive,
dup_file_timestamp = 'dup-timestamp',
flatten = flatten,
delete_empty_dirs = delete_empty_dirs)
with dir_operation_tester(extra_content_items = extra_content_items) as test:
if files:
files = [ path.join(test.src_dir, f) for f in files ]
else:
files = [ test.src_dir ]
test.result = dir_combine.combine(files, options = options)
return test
def _find_and_unsplit_test(self,
extra_content_items = None,
recursive = False,
check_downloading = _DEFAULT_FILE_SPLIT_OPTIONS.check_downloading,
check_modified = _DEFAULT_FILE_SPLIT_OPTIONS.check_modified,
check_modified_interval = _DEFAULT_FILE_SPLIT_OPTIONS.check_modified_interval,
existing_file_timestamp = None,
ignore_extensions = None,
files = None,
unzip = False):
options = file_split_options(recursive = recursive,
check_downloading = check_downloading,
existing_file_timestamp = existing_file_timestamp,
ignore_extensions = ignore_extensions,
unzip = unzip)
with dir_operation_tester(extra_content_items = extra_content_items) as test:
files = files or [ test.src_dir ]
test.result = file_split.find_and_unsplit(files, options = options)
return test
def _find_dups_test(self,
extra_content_items=None,
recursive=False,
small_checksum_size=1024 * 1024,
prefer_prefixes=None,
delete=False,
keep_empty_dirs=False):
with dir_operation_tester(
extra_content_items=extra_content_items) as test:
args = [
'file_duplicates',
'dups',
test.src_dir,
]
if recursive:
args.append('--recursive')
if delete:
args.append('--delete')
if keep_empty_dirs:
args.append('--keep')
test.result = self.run_program(self._program, args)
return test
def test_find_duplicates_with_setup(self):
items = [
temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644),
temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644),
temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644),
temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi',
0o0644),
temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi',
0o0644),
]
options = file_duplicates_options(recursive=True)
with dir_operation_tester(extra_content_items=items) as t:
setup = file_duplicates.setup([t.src_dir], options=options)
t.result = file_duplicates.find_duplicates_with_setup(setup)
self.assertEqual(
self._xp_result_item_list([
(f'{t.src_dir}/a/kiwi.jpg', [
f'{t.src_dir}/b/kiwi_dup1.jpg',
f'{t.src_dir}/c/kiwi_dup2.jpg',
]),
]), t.result.items)
