def test_prefix_filtering_consistent(self):
# The same filter should work for both local and remote files.
# So if I have a directory with 2 files:
local_files = [
self.file_info('test1.txt'),
self.file_info('nottest1.txt'),
]
# And the same 2 files remote (note that the way FileInfo objects
# are constructed, we'll have the bucket name but no leading '/'
# character):
remote_files = [
self.file_info('bucket/test1.txt', src_type='s3'),
self.file_info('bucket/nottest1.txt', src_type='s3'),
]
# If I apply the filter to the local to the local files.
exclude_filter = Filter({'filters': [['--exclude', 't*']]})
filtered_files = list(exclude_filter.call(local_files))
self.assertEqual(len(filtered_files), 1)
self.assertEqual(os.path.basename(filtered_files[0].src),
'nottest1.txt')
# I should get the same result if I apply the same filter to s3
# objects.
same_filtered_files = list(exclude_filter.call(remote_files))
self.assertEqual(len(same_filtered_files), 1)
self.assertEqual(os.path.basename(same_filtered_files[0].src),
'nottest1.txt')
def test_exclude(self):
exclude_filter = Filter({'filters': [['--exclude', '*']]})
matched_files = list(exclude_filter.call(self.local_files))
self.assertEqual(matched_files, [])
matched_files = list(exclude_filter.call(self.s3_files))
self.assertEqual(matched_files, [])
def test_no_filter(self):
exc_inc_filter = Filter({})
matched_files = list(exc_inc_filter.call(self.local_files))
self.assertEqual(matched_files, self.local_files)
matched_files2 = list(exc_inc_filter.call(self.s3_files))
self.assertEqual(matched_files2, self.s3_files)
def test_include_exclude(self):
patterns = [['--include', '*.txt'], ['--exclude', '*']]
exclude_all_filter = Filter({'filters': patterns})
matched_files = list(exclude_all_filter.call(self.local_files))
self.assertEqual(matched_files, [])
matched_files = list(exclude_all_filter.call(self.s3_files))
self.assertEqual(matched_files, [])
def test_include(self):
patterns = [['--include', '*.txt']]
include_filter = Filter({'filters': [['--include', '*.txt']]})
matched_files = list(include_filter.call(self.local_files))
self.assertEqual(matched_files, self.local_files)
matched_files2 = list(include_filter.call(self.s3_files))
self.assertEqual(matched_files2, self.s3_files)
def test_exclude(self):
"""
Only an exclude filter
"""
patterns = [['--exclude', '*']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
def test_no_filter(self):
"""
No filters
"""
patterns = []
exc_inc_filter = Filter({})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.local_files)
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.s3_files)
def test_include_exclude(self):
"""
Include all .txt files then exclude everything
"""
patterns = [['--include', '*.txt'], ['--exclude', '*']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
def test_include(self):
"""
Only an include file
"""
patterns = [['--include', '*.txt']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.local_files)
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.s3_files)
def test_no_filter(self):
"""
No filters
"""
patterns = []
exc_inc_filter = Filter({})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.local_files)
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, self.s3_files)
def test_include_exclude(self):
"""
Include all .txt files then exclude everything
"""
patterns = [['--include', '*.txt'], ['--exclude', '*']]
exc_inc_filter = Filter({'filters': patterns})
files = exc_inc_filter.call(iter(self.local_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
files = exc_inc_filter.call(iter(self.s3_files))
result_list = []
for filename in files:
result_list.append(filename)
self.assertEqual(result_list, [])
def create_filter(self, filters=None, root=None, dst_root=None):
if root is None:
root = os.getcwd()
if filters is None:
filters = {}
if dst_root is None:
dst_root = 'bucket'
return Filter(filters, root, dst_root)
def create_filter(self,
filters=None,
root=None,
dst_root=None,
parameters=None):
if root is None:
root = os.getcwd()
if filters is None:
filters = {}
if dst_root is None:
dst_root = 'bucket'
if parameters is not None:
return create_filter(parameters)
return Filter(filters, root, dst_root)
def run(self):
"""
This function wires together all of the generators and completes
the command. First a dictionary is created that is indexed first by
the command name. Then using the instruction, another dictionary
can be indexed to obtain the objects corresponding to the
particular instruction for that command. To begin the wiring,
either a ``FileFormat`` or ``TaskInfo`` object, depending on the
command, is put into a list. Then the function enters a while loop
that pops off an instruction. It then determines the object needed
and calls the call function of the object using the list as the input.
Depending on the number of objects in the input list and the number
of components in the list corresponding to the instruction, the call
method of the component can be called two different ways. If the
number of inputs is equal to the number of components a 1:1 mapping of
inputs to components is used when calling the call function. If the
there are more inputs than components, then a 2:1 mapping of inputs to
components is used where the component call method takes two inputs
instead of one. Whatever files are yielded from the call function
is appended to a list and used as the input for the next repetition
of the while loop until there are no more instructions.
"""
src = self.parameters['src']
dest = self.parameters['dest']
paths_type = self.parameters['paths_type']
files = FileFormat().format(src, dest, self.parameters)
rev_files = FileFormat().format(dest, src, self.parameters)
cmd_translation = {}
cmd_translation['locals3'] = {
'cp': 'upload',
'sync': 'upload',
'mv': 'move'
}
cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
cmd_translation['s3local'] = {
'cp': 'download',
'sync': 'download',
'mv': 'move'
}
cmd_translation['s3'] = {
'rm': 'delete',
'ls': 'list_objects',
'mb': 'make_bucket',
'rb': 'remove_bucket'
}
operation = cmd_translation[paths_type][self.cmd]
file_generator = FileGenerator(self.session, operation,
self.parameters)
rev_generator = FileGenerator(self.session, '', self.parameters)
taskinfo = [
TaskInfo(src=files['src']['path'],
src_type='s3',
operation=operation)
]
s3handler = S3Handler(self.session, self.parameters)
command_dict = {}
command_dict['sync'] = {
'setup': [files, rev_files],
'file_generator': [file_generator, rev_generator],
'filters': [Filter(self.parameters),
Filter(self.parameters)],
'comparator': [Comparator(self.parameters)],
's3_handler': [s3handler]
}
command_dict['cp'] = {
'setup': [files],
'file_generator': [file_generator],
'filters': [Filter(self.parameters)],
's3_handler': [s3handler]
}
command_dict['rm'] = {
'setup': [files],
'file_generator': [file_generator],
'filters': [Filter(self.parameters)],
's3_handler': [s3handler]
}
command_dict['mv'] = {
'setup': [files],
'file_generator': [file_generator],
'filters': [Filter(self.parameters)],
's3_handler': [s3handler]
}
command_dict['ls'] = {'setup': [taskinfo], 's3_handler': [s3handler]}
command_dict['mb'] = {'setup': [taskinfo], 's3_handler': [s3handler]}
command_dict['rb'] = {'setup': [taskinfo], 's3_handler': [s3handler]}
files = command_dict[self.cmd]['setup']
while self.instructions:
instruction = self.instructions.pop(0)
file_list = []
components = command_dict[self.cmd][instruction]
for i in range(len(components)):
if len(files) > len(components):
file_list.append(components[i].call(*files))
else:
file_list.append(components[i].call(files[i]))
files = file_list
def create_filter(self, filters=None, root=None):
if root is None:
root = os.getcwd()
if filters is None:
filters = {}
return Filter(filters, root)
