def _move_files_test(self, extra_content_items=None, recursive=False, dry_run=False, dup_file_timestamp=None, dup_file_count=None): with dir_operation_tester( extra_content_items=extra_content_items) as test: file_util.mkdir(test.dst_dir) args = [ 'files', 'move', test.src_dir, test.dst_dir, ] if recursive: args.append('--recursive') if dry_run: args.append('--dry-run') if dup_file_timestamp: args.extend(['--dup-file-timestamp', dup_file_timestamp]) if dup_file_count: args.extend(['--dup-file-count', str(dup_file_count)]) test.result = self.run_program(self._program, args) return test
def _partition_test( self, extra_content_items=None, dst_dir_same_as_src=False, recursive=False, files=None, partition_type=dir_partition_defaults.PARTITION_TYPE, partition_criteria=None, pre_test_function=None, delete_empty_dirs=dir_partition_defaults.DELETE_EMPTY_DIRS, threshold=dir_partition_defaults.THRESHOLD, flatten=dir_partition_defaults.FLATTEN): with dir_operation_tester( extra_content_items=extra_content_items, dst_dir_same_as_src=dst_dir_same_as_src) as test: options = dir_partition_options( recursive=recursive, dup_file_timestamp='dup-timestamp', partition_type=partition_type, partition_criteria=partition_criteria, delete_empty_dirs=delete_empty_dirs, threshold=threshold, flatten=flatten, dst_dir=test.dst_dir) if pre_test_function: pre_test_function(test) if files: files = [path.join(test.src_dir, f) for f in files] else: files = [test.src_dir] test.result = dir_partition.partition(files, options=options) return test
def _find_and_unsplit_test( self, extra_content_items=None, recursive=False, check_downloading=_DEFAULT_FILE_SPLIT_OPTIONS.check_downloading, check_modified=_DEFAULT_FILE_SPLIT_OPTIONS.check_modified, check_modified_interval=_DEFAULT_FILE_SPLIT_OPTIONS. check_modified_interval, dry_run=False): options = file_split_options(recursive=recursive, check_downloading=check_downloading) with dir_operation_tester( extra_content_items=extra_content_items) as test: args = [ 'file_split', 'unsplit', test.src_dir, ] if recursive: args.append('--recursive') if check_downloading: args.append('--check-downloading') if dry_run: args.append('--dry-run') test.result = self.run_program(self._program, args) return test
def test_find_file_duplicates_with_setup_and_removed_resolved_file(self): items = [ temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644), temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644), temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644), temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi', 0o0644), temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi', 0o0644), temp_content('file', 'foo/cheese/brie.jpg', 'this is kiwi', 0o0644), temp_content('file', 'foo/cheese/cheddar.jpg', 'this is cheddar', 0o0644), temp_content('file', 'foo/cheese/gouda.jpg', 'this is lemon', 0o0644), ] options = file_duplicates_options(recursive=True) with dir_operation_tester(extra_content_items=items) as t: setup = file_duplicates.setup([t.src_dir], options=options) file_util.remove(f'{t.src_dir}/a/kiwi.jpg') dups = file_duplicates.find_file_duplicates_with_setup( f'{t.tmp_dir}/foo/cheese/brie.jpg', setup) self.assertEqual([ f'{t.src_dir}/b/kiwi_dup1.jpg', f'{t.src_dir}/c/kiwi_dup2.jpg', ], dups) dups = file_duplicates.find_file_duplicates_with_setup( f'{t.tmp_dir}/foo/cheese/gouda.jpg', setup) self.assertEqual([ f'{t.src_dir}/a/lemon.jpg', ], dups)
def _call_find_file_duplicates(self, filename, extra_content_items=None, recursive=False, small_checksum_size=1024 * 1024, prefer_prefixes=None, sort_key=None, pre_test_function=None, include_empty_files=False, ignore_files=[]): options = file_duplicates_options( recursive=recursive, small_checksum_size=small_checksum_size, sort_key=sort_key, include_empty_files=include_empty_files, ignore_files=ignore_files) with dir_operation_tester( extra_content_items=extra_content_items) as test: if pre_test_function: pre_test_function(test) if prefer_prefixes: xglobals = {'test': test} prefer_prefixes = [eval(x, xglobals) for x in prefer_prefixes] options.prefer_prefixes = prefer_prefixes test.result = file_duplicates.find_file_duplicates(path.join( test.tmp_dir, filename), [test.src_dir], options=options) return test
def _split_test(self, multiplied_content_items=None, content_multiplier=1, chunk_size=250, extra_content_items=None, dst_dir_same_as_src=False, recursive=False, sort_order='filename', sort_reverse=False, pre_test_function=None, threshold=None): options = dir_split_options(chunk_size=chunk_size, prefix='chunk-', recursive=recursive, dup_file_timestamp='dup-timestamp', sort_order=sort_order, sort_reverse=sort_reverse, threshold=threshold) with dir_operation_tester( multiplied_content_items=multiplied_content_items, content_multiplier=content_multiplier, extra_content_items=extra_content_items, dst_dir_same_as_src=dst_dir_same_as_src) as test: if pre_test_function: pre_test_function(test) dir_split.split(test.src_dir, test.dst_dir, options) return test
def _test(self, items, files=[], recursive=False, dry_run=False, flatten=dir_combine_defaults.FLATTEN, delete_empty_dirs=dir_combine_defaults.DELETE_EMPTY_DIRS): with dir_operation_tester(extra_content_items=items) as test: if files: files_args = [path.join(test.src_dir, f) for f in files] else: files_args = [test.src_dir] args = [ 'dir_combine', 'combine', '--dup-file-timestamp', 'dup-timestamp', '--dup-file-count', '42', ] + files_args if recursive: args.append('--recursive') if dry_run: args.append('--dry-run') if flatten: args.append('--flatten') if delete_empty_dirs: args.append('--delete-empty-dirs') test.result = self.run_program(self._program, args) return test
def _partition_test(self, extra_content_items = None, dst_dir_same_as_src = False, recursive = False, partition_type = dir_partition_defaults.PARTITION_TYPE, dry_run = False, delete_empty_dirs = dir_partition_defaults.DELETE_EMPTY_DIRS, threshold = dir_partition_defaults.THRESHOLD, flatten = dir_partition_defaults.FLATTEN): with dir_operation_tester(extra_content_items = extra_content_items, dst_dir_same_as_src = dst_dir_same_as_src) as test: args = [ 'dir_partition', 'partition', '--type', partition_type, '--dst-dir', test.dst_dir, test.src_dir, ] if recursive: args.append('--recursive') if dry_run: args.append('--dry-run') if delete_empty_dirs: args.append('--delete-empty-dirs') if threshold: args.extend([ '--threshold', str(threshold) ]) if flatten: args.append('--flatten') test.result = self.run_program(self._program, args) return test
def _test(self, items, recursive=False, dry_run=False): with dir_operation_tester(extra_content_items=items) as test: args = [ 'dirs', 'remove_empty', test.src_dir, ] if recursive: args.append('--recursive') if dry_run: args.append('--dry-run') test.result = self.run_program(self._program, args) return test
def _split_test(self, multiplied_content_items, content_multiplier, chunk_size, extra_content_items = None): with dir_operation_tester(multiplied_content_items = multiplied_content_items, content_multiplier = content_multiplier, extra_content_items = extra_content_items) as test: args = [ 'dir_split', 'split', '--prefix', 'chunk-', '--dst-dir', test.dst_dir, '--chunk-size', str(chunk_size), test.src_dir, ] test.result = self.run_program(self._program, args) return test
def _combine_test(self, extra_content_items = None, dst_dir_same_as_src = False, recursive = False, files = None, flatten = dir_combine_defaults.FLATTEN, delete_empty_dirs = dir_combine_defaults.DELETE_EMPTY_DIRS): options = dir_combine_options(recursive = recursive, dup_file_timestamp = 'dup-timestamp', flatten = flatten, delete_empty_dirs = delete_empty_dirs) with dir_operation_tester(extra_content_items = extra_content_items) as test: if files: files = [ path.join(test.src_dir, f) for f in files ] else: files = [ test.src_dir ] test.result = dir_combine.combine(files, options = options) return test
def _find_and_unsplit_test(self, extra_content_items = None, recursive = False, check_downloading = _DEFAULT_FILE_SPLIT_OPTIONS.check_downloading, check_modified = _DEFAULT_FILE_SPLIT_OPTIONS.check_modified, check_modified_interval = _DEFAULT_FILE_SPLIT_OPTIONS.check_modified_interval, existing_file_timestamp = None, ignore_extensions = None, files = None, unzip = False): options = file_split_options(recursive = recursive, check_downloading = check_downloading, existing_file_timestamp = existing_file_timestamp, ignore_extensions = ignore_extensions, unzip = unzip) with dir_operation_tester(extra_content_items = extra_content_items) as test: files = files or [ test.src_dir ] test.result = file_split.find_and_unsplit(files, options = options) return test
def _find_dups_test(self, extra_content_items=None, recursive=False, small_checksum_size=1024 * 1024, prefer_prefixes=None, delete=False, keep_empty_dirs=False): with dir_operation_tester( extra_content_items=extra_content_items) as test: args = [ 'file_duplicates', 'dups', test.src_dir, ] if recursive: args.append('--recursive') if delete: args.append('--delete') if keep_empty_dirs: args.append('--keep') test.result = self.run_program(self._program, args) return test
def test_find_duplicates_with_setup(self): items = [ temp_content('file', 'src/a/kiwi.jpg', 'this is kiwi', 0o0644), temp_content('file', 'src/a/apple.jpg', 'this is apple', 0o0644), temp_content('file', 'src/a/lemon.jpg', 'this is lemon', 0o0644), temp_content('file', 'src/b/kiwi_dup1.jpg', 'this is kiwi', 0o0644), temp_content('file', 'src/c/kiwi_dup2.jpg', 'this is kiwi', 0o0644), ] options = file_duplicates_options(recursive=True) with dir_operation_tester(extra_content_items=items) as t: setup = file_duplicates.setup([t.src_dir], options=options) t.result = file_duplicates.find_duplicates_with_setup(setup) self.assertEqual( self._xp_result_item_list([ (f'{t.src_dir}/a/kiwi.jpg', [ f'{t.src_dir}/b/kiwi_dup1.jpg', f'{t.src_dir}/c/kiwi_dup2.jpg', ]), ]), t.result.items)