def test_s3_delete_directory(self):
"""
Generates s3 files under a common prefix. Also it ensures that
the directory itself is included because it is a delete command
Note: Size and last update are not tested because s3 generates them.
"""
input_s3_file = {
'src': {
'path': self.bucket + '/',
'type': 's3'
},
'dest': {
'path': '',
'type': 'local'
},
'dir_op': True,
'use_src_name': True
}
params = {'region': 'us-east-1'}
files = FileGenerator(self.session, 'delete',
params).call(input_s3_file)
result_list = []
for filename in files:
result_list.append(filename)
file_info1 = FileInfo(src=self.bucket + '/another_directory/',
dest='another_directory' + os.sep,
compare_key='another_directory/',
size=result_list[0].size,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local',
operation='delete')
file_info2 = FileInfo(src=self.file2,
dest='another_directory' + os.sep + 'text2.txt',
compare_key='another_directory/text2.txt',
size=result_list[1].size,
last_update=result_list[1].last_update,
src_type='s3',
dest_type='local',
operation='delete')
file_info3 = FileInfo(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=result_list[2].size,
last_update=result_list[2].last_update,
src_type='s3',
dest_type='local',
operation='delete')
ref_list = [file_info1, file_info2, file_info3]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_s3_delete_directory(self):
#
# Generates s3 files under a common prefix. Also it ensures that
# the directory itself is included because it is a delete command
# Note: Size and last update are not tested because s3 generates them.
#
input_s3_file = {
'src': {
'path': self.bucket + '/',
'type': 's3'
},
'dest': {
'path': '',
'type': 'local'
},
'dir_op': True,
'use_src_name': True
}
result_list = list(
FileGenerator(self.service, self.endpoint,
'delete').call(input_s3_file))
file_stat1 = FileStat(src=self.bucket + '/another_directory/',
dest='another_directory' + os.sep,
compare_key='another_directory/',
size=0,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local',
operation_name='delete')
file_stat2 = FileStat(src=self.file2,
dest='another_directory' + os.sep + 'text2.txt',
compare_key='another_directory/text2.txt',
size=21,
last_update=result_list[1].last_update,
src_type='s3',
dest_type='local',
operation_name='delete')
file_stat3 = FileStat(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=15,
last_update=result_list[2].last_update,
src_type='s3',
dest_type='local',
operation_name='delete')
expected_list = [file_stat1, file_stat2, file_stat3]
self.assertEqual(len(result_list), 3)
compare_files(self, result_list[0], expected_list[0])
compare_files(self, result_list[1], expected_list[1])
compare_files(self, result_list[2], expected_list[2])
def test_error_raised_on_decoding_error(self, listdir_mock):
# On Python3, sys.getdefaultencoding
file_generator = FileGenerator(None, None, None)
# utf-8 encoding for U+2713.
listdir_mock.return_value = [b'\xe2\x9c\x93']
list(file_generator.list_files(self.directory, dir_op=True))
# Ensure the message was added to the result queue and is
# being skipped.
self.assertFalse(file_generator.result_queue.empty())
warning_message = file_generator.result_queue.get()
self.assertIn("warning: Skipping file ", warning_message.message)
self.assertIn("Please check your locale settings.",
warning_message.message)
def test_no_read_access(self):
file_gen = FileGenerator(self.client, '', False)
self.files.create_file("foo.txt", contents="foo")
full_path = os.path.join(self.root, "foo.txt")
open_function = 'awscli.customizations.s3.filegenerator._open'
with mock.patch(open_function) as mock_class:
mock_class.side_effect = OSError()
return_val = file_gen.triggers_warning(full_path)
self.assertTrue(return_val)
warning_message = file_gen.result_queue.get()
self.assertEqual(warning_message.message,
("warning: Skipping file %s. File/Directory is "
"not readable." % full_path))
def test_is_special_file_warning(self):
file_gen = FileGenerator(self.client, '', False)
file_path = os.path.join(self.files.rootdir, 'foo')
# Use socket for special file.
sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
sock.bind(file_path)
return_val = file_gen.triggers_warning(file_path)
self.assertTrue(return_val)
warning_message = file_gen.result_queue.get()
self.assertEqual(warning_message.message,
("warning: Skipping file %s. File is character "
"special device, block special device, FIFO, or "
"socket." % file_path))
def test_list_files_is_in_sorted_order(self):
p = os.path.join
open(p(self.directory, 'test-123.txt'), 'w').close()
open(p(self.directory, 'test-321.txt'), 'w').close()
open(p(self.directory, 'test123.txt'), 'w').close()
open(p(self.directory, 'test321.txt'), 'w').close()
os.mkdir(p(self.directory, 'test'))
open(p(self.directory, 'test', 'foo.txt'), 'w').close()
file_generator = FileGenerator(None, None, None, None)
values = list(
el[0]
for el in file_generator.list_files(self.directory, dir_op=True))
self.assertEqual(values, list(sorted(values)))
def test_local_directory(self):
"""
Generate an entire local directory.
"""
input_local_dir = {
'src': {
'path': self.local_dir,
'type': 'local'
},
'dest': {
'path': 'bucket/',
'type': 's3'
},
'dir_op': True,
'use_src_name': True
}
params = {'region': 'us-east-1'}
files = FileGenerator(self.service, self.endpoint, '',
params).call(input_local_dir)
result_list = []
for filename in files:
result_list.append(filename)
size, last_update = get_file_stat(self.local_file)
file_info = FileInfo(src=self.local_file,
dest='bucket/text1.txt',
compare_key='text1.txt',
size=size,
last_update=last_update,
src_type='local',
dest_type='s3',
operation_name='',
service=None,
endpoint=None)
path = self.local_dir + 'another_directory' + os.sep \
+ 'text2.txt'
size, last_update = get_file_stat(path)
file_info2 = FileInfo(src=path,
dest='bucket/another_directory/text2.txt',
compare_key='another_directory/text2.txt',
size=size,
last_update=last_update,
src_type='local',
dest_type='s3',
operation_name='',
service=None,
endpoint=None)
ref_list = [file_info2, file_info]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_s3_directory(self):
"""
Generates s3 files under a common prefix. Also it ensures that
zero size files are ignored.
Note: Size and last update are not tested because s3 generates them.
"""
input_s3_file = {
'src': {
'path': self.bucket + '/',
'type': 's3'
},
'dest': {
'path': '',
'type': 'local'
},
'dir_op': True,
'use_src_name': True
}
params = {'region': 'us-east-1'}
files = FileGenerator(self.service, self.endpoint, '',
params).call(input_s3_file)
result_list = []
for filename in files:
result_list.append(filename)
file_info = FileInfo(src=self.file2,
dest='another_directory' + os.sep + 'text2.txt',
compare_key='another_directory/text2.txt',
size=result_list[0].size,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local',
operation_name='',
service=None,
endpoint=None)
file_info2 = FileInfo(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=result_list[1].size,
last_update=result_list[1].last_update,
src_type='s3',
dest_type='local',
operation_name='',
service=None,
endpoint=None)
ref_list = [file_info, file_info2]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_s3_delete_directory(self):
"""
Generates s3 files under a common prefix. Also it ensures that
the directory itself is included because it is a delete command
Note: Size and last update are not tested because s3 generates them.
"""
input_s3_file = {'src': {'path': self.bucket + '/', 'type': 's3'},
'dest': {'path': '', 'type': 'local'},
'dir_op': True, 'use_src_name': True}
self.parsed_responses = [{
"CommonPrefixes": [], "Contents": [
{"Key": "another_directory/", "Size": 0,
"LastModified": "2012-01-09T20:45:49.000Z"},
{"Key": "another_directory/text2.txt", "Size": 100,
"LastModified": "2014-01-09T20:45:49.000Z"},
{"Key": "text1.txt", "Size": 10,
"LastModified": "2013-01-09T20:45:49.000Z"}]}]
self.patch_make_request()
files = FileGenerator(self.client, 'delete').call(input_s3_file)
result_list = []
for filename in files:
result_list.append(filename)
file_stat1 = FileStat(src=self.bucket + '/another_directory/',
dest='another_directory' + os.sep,
compare_key='another_directory/',
size=result_list[0].size,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local', operation_name='delete')
file_stat2 = FileStat(src=self.file2,
dest='another_directory' + os.sep + 'text2.txt',
compare_key='another_directory/text2.txt',
size=result_list[1].size,
last_update=result_list[1].last_update,
src_type='s3',
dest_type='local', operation_name='delete')
file_stat3 = FileStat(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=result_list[2].size,
last_update=result_list[2].last_update,
src_type='s3',
dest_type='local', operation_name='delete')
ref_list = [file_stat1, file_stat2, file_stat3]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_warn_bad_symlink(self):
"""
This tests to make sure it fails when following bad symlinks.
"""
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_stats = FileGenerator(self.client, '', True).call(input_local_dir)
file_gen = FileGenerator(self.client, '', True)
file_stats = file_gen.call(input_local_dir)
all_filenames = self.filenames + self.symlink_files
all_filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(all_filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(all_filenames[i]))
self.assertEqual(result_list[i], filename)
self.assertFalse(file_gen.result_queue.empty())
def test_s3_single_file_404(self):
"""
Test the error message for a 404 ClientError for a single file listing
"""
input_s3_file = {'src': {'path': self.file1, 'type': 's3'},
'dest': {'path': 'text1.txt', 'type': 'local'},
'dir_op': False, 'use_src_name': False}
params = {'region': 'us-east-1'}
self.client = mock.Mock()
self.client.head_object.side_effect = \
ClientError(404, 'Not Found', '404', 'HeadObject', 404)
file_gen = FileGenerator(self.client, '')
files = file_gen.call(input_s3_file)
# The error should include 404 and should include the key name.
with self.assertRaisesRegexp(ClientError, '404.*text1.txt'):
list(files)
def test_no_follow_symlink(self):
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_stats = FileGenerator(self.client, '', False).call(input_local_dir)
self.filenames.sort()
result_list = []
for file_stat in file_stats:
result_list.append(getattr(file_stat, 'src'))
self.assertEqual(len(result_list), len(self.filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(self.filenames[i]))
self.assertEqual(result_list[i], filename)
def test_page_size(self):
input_s3_file = {'src': {'path': self.bucket+'/', 'type': 's3'},
'dest': {'path': '', 'type': 'local'},
'dir_op': True, 'use_src_name': True}
file_gen = FileGenerator(self.service, self.endpoint, '',
page_size=1).call(input_s3_file)
limited_file_gen = itertools.islice(file_gen, 1)
result_list = list(limited_file_gen)
file_stat = FileStat(src=self.file2,
dest='another_directory' + os.sep + 'text2.txt',
compare_key='another_directory/text2.txt',
size=21,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local', operation_name='')
# Ensure only one item is returned from ``ListObjects``
self.assertEqual(len(result_list), 1)
compare_files(self, result_list[0], file_stat)
def test_s3_single_file_delete(self):
input_s3_file = {'src': {'path': self.file1, 'type': 's3'},
'dest': {'path': '', 'type': 'local'},
'dir_op': False, 'use_src_name': True}
self.client = mock.Mock()
file_gen = FileGenerator(self.client, 'delete')
result_list = list(file_gen.call(input_s3_file))
self.assertEqual(len(result_list), 1)
compare_files(
self,
result_list[0],
FileStat(src=self.file1, dest='text1.txt',
compare_key='text1.txt',
size=None, last_update=None,
src_type='s3', dest_type='local',
operation_name='delete')
)
self.client.head_object.assert_not_called()
def test_s3_directory(self):
#
# Generates s3 files under a common prefix. Also it ensures that
# zero size files are ignored.
# Note: Size and last update are not tested because s3 generates them.
#
input_s3_file = {
'src': {
'path': self.bucket + '/',
'type': 's3'
},
'dest': {
'path': '',
'type': 'local'
},
'dir_op': True,
'use_src_name': True
}
params = {'region': 'us-east-1'}
result_list = list(
FileGenerator(self.service, self.endpoint, '',
params).call(input_s3_file))
file_info = FileInfo(src=self.file2,
dest='another_directory' + os.sep + 'text2.txt',
compare_key='another_directory/text2.txt',
size=21,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local',
operation_name='')
file_info2 = FileInfo(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=15,
last_update=result_list[1].last_update,
src_type='s3',
dest_type='local',
operation_name='')
expected_result = [file_info, file_info2]
self.assertEqual(len(result_list), 2)
compare_files(self, result_list[0], expected_result[0])
compare_files(self, result_list[1], expected_result[1])
def test_s3_file(self):
"""
Generate a single s3 file
Note: Size and last update are not tested because s3 generates them.
"""
input_s3_file = {
'src': {
'path': self.file1,
'type': 's3'
},
'dest': {
'path': 'text1.txt',
'type': 'local'
},
'dir_op': False,
'use_src_name': False
}
params = {'region': 'us-east-1'}
self.parsed_responses = [{
"ETag": "abcd",
"ContentLength": 100,
"LastModified": "2014-01-09T20:45:49.000Z"
}]
self.patch_make_request()
file_gen = FileGenerator(self.client, '')
files = file_gen.call(input_s3_file)
result_list = []
for filename in files:
result_list.append(filename)
file_stat = FileStat(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=result_list[0].size,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local',
operation_name='')
ref_list = [file_stat]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_nonexist_s3_file(self):
"""
This tests to make sure that files are not misproperly yielded by
ensuring the file prefix is the exact same as what was inputted.
"""
input_s3_file = {
'src': {
'path': self.file1[:-1],
'type': 's3'
},
'dest': {
'path': 'text1.txt',
'type': 'local'
},
'dir_op': False,
'use_src_name': False
}
params = {'region': 'us-east-1'}
files = FileGenerator(self.session, '', params).call(input_s3_file)
self.assertEqual(len(list(files)), 0)
def test_follow_symlink(self):
# First remove the bad symlink.
os.remove(os.path.join(self.root, 'symlink_2'))
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_infos = FileGenerator(self.service, self.endpoint,
'', True).call(input_local_dir)
all_filenames = self.filenames + self.symlink_files
all_filenames.sort()
result_list = []
for file_info in file_infos:
result_list.append(getattr(file_info, 'src'))
self.assertEqual(len(result_list), len(all_filenames))
# Just check to make sure the right local files are generated.
for i in range(len(result_list)):
filename = six.text_type(os.path.abspath(all_filenames[i]))
self.assertEqual(result_list[i], filename)
def test_s3_file(self):
#
# Generate a single s3 file
# Note: Size and last update are not tested because s3 generates them.
#
input_s3_file = {'src': {'path': self.file1, 'type': 's3'},
'dest': {'path': 'text1.txt', 'type': 'local'},
'dir_op': False, 'use_src_name': False}
expected_file_size = 15
result_list = list(
FileGenerator(self.service, self.endpoint, '').call(
input_s3_file))
file_stat = FileStat(src=self.file1, dest='text1.txt',
compare_key='text1.txt',
size=expected_file_size,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local', operation_name='')
expected_list = [file_stat]
self.assertEqual(len(result_list), 1)
compare_files(self, result_list[0], expected_list[0])
def test_s3_file(self):
"""
Generate a single s3 file
Note: Size and last update are not tested because s3 generates them.
"""
input_s3_file = {
'src': {
'path': self.file1,
'type': 's3'
},
'dest': {
'path': 'text1.txt',
'type': 'local'
},
'dir_op': False,
'use_src_name': False
}
params = {'region': 'us-east-1'}
files = FileGenerator(self.service, self.endpoint, '',
params).call(input_s3_file)
result_list = []
for filename in files:
result_list.append(filename)
file_info = FileInfo(src=self.file1,
dest='text1.txt',
compare_key='text1.txt',
size=result_list[0].size,
last_update=result_list[0].last_update,
src_type='s3',
dest_type='local',
operation_name='',
service=None,
endpoint=None)
ref_list = [file_info]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_local_file(self):
"""
Generate a single local file.
"""
input_local_file = {'src': {'path': self.local_file,
'type': 'local'},
'dest': {'path': 'bucket/text1.txt',
'type': 's3'},
'dir_op': False, 'use_src_name': False}
params = {'region': 'us-east-1'}
files = FileGenerator(self.client, '').call(input_local_file)
result_list = []
for filename in files:
result_list.append(filename)
size, last_update = get_file_stat(self.local_file)
file_stat = FileStat(src=self.local_file, dest='bucket/text1.txt',
compare_key='text1.txt', size=size,
last_update=last_update, src_type='local',
dest_type='s3', operation_name='')
ref_list = [file_stat]
self.assertEqual(len(result_list), len(ref_list))
for i in range(len(result_list)):
compare_files(self, result_list[i], ref_list[i])
def test_follow_bad_symlink(self):
"""
This tests to make sure it fails when following bad symlinks.
"""
abs_root = six.text_type(os.path.abspath(self.root) + os.sep)
input_local_dir = {'src': {'path': abs_root,
'type': 'local'},
'dest': {'path': self.bucket,
'type': 's3'},
'dir_op': True, 'use_src_name': True}
file_infos = FileGenerator(self.service, self.endpoint,
'', True).call(input_local_dir)
result_list = []
rc = 0
try:
for file_info in file_infos:
result_list.append(getattr(file_info, 'src'))
rc = 1
except OSError as e:
pass
# Error shows up as ValueError in Python 3.
except ValueError as e:
pass
self.assertEquals(0, rc)
def run(self):
"""
This function wires together all of the generators and completes
the command. First a dictionary is created that is indexed first by
the command name. Then using the instruction, another dictionary
can be indexed to obtain the objects corresponding to the
particular instruction for that command. To begin the wiring,
either a ``FileFormat`` or ``TaskInfo`` object, depending on the
command, is put into a list. Then the function enters a while loop
that pops off an instruction. It then determines the object needed
and calls the call function of the object using the list as the input.
Depending on the number of objects in the input list and the number
of components in the list corresponding to the instruction, the call
method of the component can be called two different ways. If the
number of inputs is equal to the number of components a 1:1 mapping of
inputs to components is used when calling the call function. If the
there are more inputs than components, then a 2:1 mapping of inputs to
components is used where the component call method takes two inputs
instead of one. Whatever files are yielded from the call function
is appended to a list and used as the input for the next repetition
of the while loop until there are no more instructions.
"""
src = self.parameters['src']
dest = self.parameters['dest']
paths_type = self.parameters['paths_type']
files = FileFormat().format(src, dest, self.parameters)
rev_files = FileFormat().format(dest, src, self.parameters)
cmd_translation = {}
cmd_translation['locals3'] = {
'cp': 'upload',
'sync': 'upload',
'mv': 'move'
}
cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
cmd_translation['s3local'] = {
'cp': 'download',
'sync': 'download',
'mv': 'move'
}
cmd_translation['s3'] = {
'rm': 'delete',
'mb': 'make_bucket',
'rb': 'remove_bucket'
}
operation_name = cmd_translation[paths_type][self.cmd]
file_generator = FileGenerator(self._service, self._endpoint,
operation_name,
self.parameters['follow_symlinks'])
rev_generator = FileGenerator(self._service, self._endpoint, '',
self.parameters['follow_symlinks'])
taskinfo = [
TaskInfo(src=files['src']['path'],
src_type='s3',
operation_name=operation_name,
service=self._service,
endpoint=self._endpoint)
]
s3handler = S3Handler(self.session, self.parameters)
command_dict = {}
if self.cmd == 'sync':
command_dict = {
'setup': [files, rev_files],
'file_generator': [file_generator, rev_generator],
'filters': [
create_filter(self.parameters),
create_filter(self.parameters)
],
'comparator': [Comparator(self.parameters)],
's3_handler': [s3handler]
}
elif self.cmd == 'cp':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
's3_handler': [s3handler]
}
elif self.cmd == 'rm':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
's3_handler': [s3handler]
}
elif self.cmd == 'mv':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
's3_handler': [s3handler]
}
elif self.cmd == 'mb':
command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
elif self.cmd == 'rb':
command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
files = command_dict['setup']
while self.instructions:
instruction = self.instructions.pop(0)
file_list = []
components = command_dict[instruction]
for i in range(len(components)):
if len(files) > len(components):
file_list.append(components[i].call(*files))
else:
file_list.append(components[i].call(files[i]))
files = file_list
# This is kinda quirky, but each call through the instructions
# will replaces the files attr with the return value of the
# file_list. The very last call is a single list of
# [s3_handler], and the s3_handler returns the number of
# tasks failed. This means that files[0] now contains
# the number of failed tasks. In terms of the RC, we're
# keeping it simple and saying that > 0 failed tasks
# will give a 1 RC.
rc = 0
if files[0] > 0:
rc = 1
return rc
def test_list_files_with_invalid_timestamp(self, stat_mock):
stat_mock.return_value = 9, None
open(os.path.join(self.directory, 'test'), 'w').close()
file_generator = FileGenerator(None, None, None)
value = list(file_generator.list_files(self.directory, dir_op=True))[0]
self.assertIs(value[1]['LastModified'], EPOCH_TIME)
def test_bad_symlink(self):
path = os.path.join(self.files.rootdir, 'badsymlink')
os.symlink('non-existent-file', path)
filegenerator = FileGenerator(self.service, self.endpoint,
'', True)
self.assertFalse(filegenerator.should_ignore_file(path))
def test_warning(self):
path = os.path.join(self.files.rootdir, 'badsymlink')
os.symlink('non-existent-file', path)
filegenerator = FileGenerator(self.client, '', True)
self.assertTrue(filegenerator.should_ignore_file(path))
def run(self):
"""
This function wires together all of the generators and completes
the command. First a dictionary is created that is indexed first by
the command name. Then using the instruction, another dictionary
can be indexed to obtain the objects corresponding to the
particular instruction for that command. To begin the wiring,
either a ``FileFormat`` or ``TaskInfo`` object, depending on the
command, is put into a list. Then the function enters a while loop
that pops off an instruction. It then determines the object needed
and calls the call function of the object using the list as the input.
Depending on the number of objects in the input list and the number
of components in the list corresponding to the instruction, the call
method of the component can be called two different ways. If the
number of inputs is equal to the number of components a 1:1 mapping of
inputs to components is used when calling the call function. If the
there are more inputs than components, then a 2:1 mapping of inputs to
components is used where the component call method takes two inputs
instead of one. Whatever files are yielded from the call function
is appended to a list and used as the input for the next repetition
of the while loop until there are no more instructions.
"""
src = self.parameters['src']
dest = self.parameters['dest']
paths_type = self.parameters['paths_type']
files = FileFormat().format(src, dest, self.parameters)
rev_files = FileFormat().format(dest, src, self.parameters)
cmd_translation = {}
cmd_translation['locals3'] = {
'cp': 'upload',
'sync': 'upload',
'mv': 'move'
}
cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
cmd_translation['s3local'] = {
'cp': 'download',
'sync': 'download',
'mv': 'move'
}
cmd_translation['s3'] = {
'rm': 'delete',
'mb': 'make_bucket',
'rb': 'remove_bucket'
}
result_queue = queue.Queue()
operation_name = cmd_translation[paths_type][self.cmd]
fgen_kwargs = {
'client': self._source_client,
'operation_name': operation_name,
'follow_symlinks': self.parameters['follow_symlinks'],
'page_size': self.parameters['page_size'],
'result_queue': result_queue
}
rgen_kwargs = {
'client': self._client,
'operation_name': '',
'follow_symlinks': self.parameters['follow_symlinks'],
'page_size': self.parameters['page_size'],
'result_queue': result_queue
}
fgen_request_parameters = {}
fgen_head_object_params = {}
fgen_request_parameters['HeadObject'] = fgen_head_object_params
fgen_kwargs['request_parameters'] = fgen_request_parameters
# SSE-C may be neaded for HeadObject for copies/downloads/deletes
# If the operation is s3 to s3, the FileGenerator should use the
# copy source key and algorithm. Otherwise, use the regular
# SSE-C key and algorithm. Note the reverse FileGenerator does
# not need any of these because it is used only for sync operations
# which only use ListObjects which does not require HeadObject.
RequestParamsMapper.map_head_object_params(fgen_head_object_params,
self.parameters)
if paths_type == 's3s3':
RequestParamsMapper.map_head_object_params(
fgen_head_object_params, {
'sse_c': self.parameters.get('sse_c_copy_source'),
'sse_c_key': self.parameters.get('sse_c_copy_source_key')
})
file_generator = FileGenerator(**fgen_kwargs)
rev_generator = FileGenerator(**rgen_kwargs)
taskinfo = [
TaskInfo(src=files['src']['path'],
src_type='s3',
operation_name=operation_name,
client=self._client)
]
stream_dest_path, stream_compare_key = find_dest_path_comp_key(files)
stream_file_info = [
FileInfo(src=files['src']['path'],
dest=stream_dest_path,
compare_key=stream_compare_key,
src_type=files['src']['type'],
dest_type=files['dest']['type'],
operation_name=operation_name,
client=self._client,
is_stream=True)
]
file_info_builder = FileInfoBuilder(self._client, self._source_client,
self.parameters)
s3handler = S3Handler(self.session,
self.parameters,
runtime_config=self._runtime_config,
result_queue=result_queue)
s3_stream_handler = S3StreamHandler(self.session,
self.parameters,
result_queue=result_queue)
sync_strategies = self.choose_sync_strategies()
command_dict = {}
if self.cmd == 'sync':
command_dict = {
'setup': [files, rev_files],
'file_generator': [file_generator, rev_generator],
'filters': [
create_filter(self.parameters),
create_filter(self.parameters)
],
'comparator': [Comparator(**sync_strategies)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'cp' and self.parameters['is_stream']:
command_dict = {
'setup': [stream_file_info],
's3_handler': [s3_stream_handler]
}
elif self.cmd == 'cp':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'rm':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'mv':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'mb':
command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
elif self.cmd == 'rb':
command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
files = command_dict['setup']
while self.instructions:
instruction = self.instructions.pop(0)
file_list = []
components = command_dict[instruction]
for i in range(len(components)):
if len(files) > len(components):
file_list.append(components[i].call(*files))
else:
file_list.append(components[i].call(files[i]))
files = file_list
# This is kinda quirky, but each call through the instructions
# will replaces the files attr with the return value of the
# file_list. The very last call is a single list of
# [s3_handler], and the s3_handler returns the number of
# tasks failed and the number of tasks warned.
# This means that files[0] now contains a namedtuple with
# the number of failed tasks and the number of warned tasks.
# In terms of the RC, we're keeping it simple and saying
# that > 0 failed tasks will give a 1 RC and > 0 warned
# tasks will give a 2 RC. Otherwise a RC of zero is returned.
rc = 0
if files[0].num_tasks_failed > 0:
rc = 1
if files[0].num_tasks_warned > 0:
rc = 2
return rc
def run(self):
"""
This function wires together all of the generators and completes
the command. First a dictionary is created that is indexed first by
the command name. Then using the instruction, another dictionary
can be indexed to obtain the objects corresponding to the
particular instruction for that command. To begin the wiring,
either a ``FileFormat`` or ``TaskInfo`` object, depending on the
command, is put into a list. Then the function enters a while loop
that pops off an instruction. It then determines the object needed
and calls the call function of the object using the list as the input.
Depending on the number of objects in the input list and the number
of components in the list corresponding to the instruction, the call
method of the component can be called two different ways. If the
number of inputs is equal to the number of components a 1:1 mapping of
inputs to components is used when calling the call function. If the
there are more inputs than components, then a 2:1 mapping of inputs to
components is used where the component call method takes two inputs
instead of one. Whatever files are yielded from the call function
is appended to a list and used as the input for the next repetition
of the while loop until there are no more instructions.
"""
src = self.parameters['src']
dest = self.parameters['dest']
paths_type = self.parameters['paths_type']
files = FileFormat().format(src, dest, self.parameters)
rev_files = FileFormat().format(dest, src, self.parameters)
cmd_translation = {}
cmd_translation['locals3'] = {
'cp': 'upload',
'sync': 'upload',
'mv': 'move'
}
cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
cmd_translation['s3local'] = {
'cp': 'download',
'sync': 'download',
'mv': 'move'
}
cmd_translation['s3'] = {
'rm': 'delete',
'ls': 'list_objects',
'mb': 'make_bucket',
'rb': 'remove_bucket'
}
operation = cmd_translation[paths_type][self.cmd]
file_generator = FileGenerator(self.session, operation,
self.parameters)
rev_generator = FileGenerator(self.session, '', self.parameters)
taskinfo = [
TaskInfo(src=files['src']['path'],
src_type='s3',
operation=operation)
]
s3handler = S3Handler(self.session, self.parameters)
command_dict = {}
command_dict['sync'] = {
'setup': [files, rev_files],
'file_generator': [file_generator, rev_generator],
'filters': [Filter(self.parameters),
Filter(self.parameters)],
'comparator': [Comparator(self.parameters)],
's3_handler': [s3handler]
}
command_dict['cp'] = {
'setup': [files],
'file_generator': [file_generator],
'filters': [Filter(self.parameters)],
's3_handler': [s3handler]
}
command_dict['rm'] = {
'setup': [files],
'file_generator': [file_generator],
'filters': [Filter(self.parameters)],
's3_handler': [s3handler]
}
command_dict['mv'] = {
'setup': [files],
'file_generator': [file_generator],
'filters': [Filter(self.parameters)],
's3_handler': [s3handler]
}
command_dict['ls'] = {'setup': [taskinfo], 's3_handler': [s3handler]}
command_dict['mb'] = {'setup': [taskinfo], 's3_handler': [s3handler]}
command_dict['rb'] = {'setup': [taskinfo], 's3_handler': [s3handler]}
files = command_dict[self.cmd]['setup']
while self.instructions:
instruction = self.instructions.pop(0)
file_list = []
components = command_dict[self.cmd][instruction]
for i in range(len(components)):
if len(files) > len(components):
file_list.append(components[i].call(*files))
else:
file_list.append(components[i].call(files[i]))
files = file_list
