def test_prefix_filtering_consistent(self):
# The same filter should work for both local and remote files.
# So if I have a directory with 2 files:
local_files = [
self.file_info('test1.txt'),
self.file_info('nottest1.txt'),
]
# And the same 2 files remote (note that the way FileInfo objects
# are constructed, we'll have the bucket name but no leading '/'
# character):
remote_files = [
self.file_info('bucket/test1.txt', src_type='s3'),
self.file_info('bucket/nottest1.txt', src_type='s3'),
]
# If I apply the filter to the local to the local files.
exclude_filter = Filter({'filters': [['--exclude', 't*']]})
filtered_files = list(exclude_filter.call(local_files))
self.assertEqual(len(filtered_files), 1)
self.assertEqual(os.path.basename(filtered_files[0].src),
'nottest1.txt')
# I should get the same result if I apply the same filter to s3
# objects.
same_filtered_files = list(exclude_filter.call(remote_files))
self.assertEqual(len(same_filtered_files), 1)
self.assertEqual(os.path.basename(same_filtered_files[0].src),
'nottest1.txt')
def test_exclude(self):
exclude_filter = Filter({'filters': [['--exclude', '*']]})
matched_files = list(exclude_filter.call(self.local_files))
self.assertEqual(matched_files, [])
matched_files = list(exclude_filter.call(self.s3_files))
self.assertEqual(matched_files, [])
def test_no_filter(self):
exc_inc_filter = Filter({})
matched_files = list(exc_inc_filter.call(self.local_files))
self.assertEqual(matched_files, self.local_files)
matched_files2 = list(exc_inc_filter.call(self.s3_files))
self.assertEqual(matched_files2, self.s3_files)
def test_include_exclude(self):
patterns = [['--include', '*.txt'], ['--exclude', '*']]
exclude_all_filter = Filter({'filters': patterns})
matched_files = list(exclude_all_filter.call(self.local_files))
self.assertEqual(matched_files, [])
matched_files = list(exclude_all_filter.call(self.s3_files))
self.assertEqual(matched_files, [])
def test_include(self):
patterns = [['--include', '*.txt']]
include_filter = Filter({'filters': [['--include', '*.txt']]})
matched_files = list(include_filter.call(self.local_files))
self.assertEqual(matched_files, self.local_files)
matched_files2 = list(include_filter.call(self.s3_files))
self.assertEqual(matched_files2, self.s3_files)
def test_exclude(self):
"""
Only an exclude filter
"""
patterns = [['--exclude', '*']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
def test_no_filter(self):
"""
No filters
"""
patterns = []
exc_inc_filter = Filter({})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.local_files)
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.s3_files)
def test_include_exclude(self):
"""
Include all .txt files then exclude everything
"""
patterns = [['--include', '*.txt'], ['--exclude', '*']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
def test_include(self):
"""
Only an include file
"""
patterns = [['--include', '*.txt']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.local_files)
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.s3_files)
def test_no_filter(self):
"""
No filters
"""
patterns = []
exc_inc_filter = Filter({})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.local_files)
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.s3_files)
def test_include_exclude(self):
"""
Include all .txt files then exclude everything
"""
patterns = [['--include', '*.txt'], ['--exclude', '*']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
