def test_prefix_filtering_consistent(self): # The same filter should work for both local and remote files. # So if I have a directory with 2 files: local_files = [ self.file_info('test1.txt'), self.file_info('nottest1.txt'), ] # And the same 2 files remote (note that the way FileInfo objects # are constructed, we'll have the bucket name but no leading '/' # character): remote_files = [ self.file_info('bucket/test1.txt', src_type='s3'), self.file_info('bucket/nottest1.txt', src_type='s3'), ] # If I apply the filter to the local to the local files. exclude_filter = Filter({'filters': [['--exclude', 't*']]}) filtered_files = list(exclude_filter.call(local_files)) self.assertEqual(len(filtered_files), 1) self.assertEqual(os.path.basename(filtered_files[0].src), 'nottest1.txt') # I should get the same result if I apply the same filter to s3 # objects. same_filtered_files = list(exclude_filter.call(remote_files)) self.assertEqual(len(same_filtered_files), 1) self.assertEqual(os.path.basename(same_filtered_files[0].src), 'nottest1.txt')
def test_exclude(self): exclude_filter = Filter({'filters': [['--exclude', '*']]}) matched_files = list(exclude_filter.call(self.local_files)) self.assertEqual(matched_files, []) matched_files = list(exclude_filter.call(self.s3_files)) self.assertEqual(matched_files, [])
def test_no_filter(self): exc_inc_filter = Filter({}) matched_files = list(exc_inc_filter.call(self.local_files)) self.assertEqual(matched_files, self.local_files) matched_files2 = list(exc_inc_filter.call(self.s3_files)) self.assertEqual(matched_files2, self.s3_files)
def test_include_exclude(self): patterns = [['--include', '*.txt'], ['--exclude', '*']] exclude_all_filter = Filter({'filters': patterns}) matched_files = list(exclude_all_filter.call(self.local_files)) self.assertEqual(matched_files, []) matched_files = list(exclude_all_filter.call(self.s3_files)) self.assertEqual(matched_files, [])
def test_include(self): patterns = [['--include', '*.txt']] include_filter = Filter({'filters': [['--include', '*.txt']]}) matched_files = list(include_filter.call(self.local_files)) self.assertEqual(matched_files, self.local_files) matched_files2 = list(include_filter.call(self.s3_files)) self.assertEqual(matched_files2, self.s3_files)
def test_exclude(self): """ Only an exclude filter """ patterns = [['--exclude', '*']] exc_inc_filter = Filter({'filters': patterns}) files = exc_inc_filter.call(iter(self.local_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, []) files = exc_inc_filter.call(iter(self.s3_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, [])
def test_no_filter(self): """ No filters """ patterns = [] exc_inc_filter = Filter({}) files = exc_inc_filter.call(iter(self.local_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, self.local_files) files = exc_inc_filter.call(iter(self.s3_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, self.s3_files)
def test_include_exclude(self): """ Include all .txt files then exclude everything """ patterns = [['--include', '*.txt'], ['--exclude', '*']] exc_inc_filter = Filter({'filters': patterns}) files = exc_inc_filter.call(iter(self.local_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, []) files = exc_inc_filter.call(iter(self.s3_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, [])
def test_include(self): """ Only an include file """ patterns = [['--include', '*.txt']] exc_inc_filter = Filter({'filters': patterns}) files = exc_inc_filter.call(iter(self.local_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, self.local_files) files = exc_inc_filter.call(iter(self.s3_files)) result_list = [] for filename in files: result_list.append(filename) self.assertEqual(result_list, self.s3_files)