def _download(self, manager, bucket, key):
"""
Download the specified object and print it to stdout.
:type manager: s3transfer.manager.TransferManager
:param manager: The transfer manager to use for the download.
:type bucket: str
:param bucket: The bucket to download the object from.
:type key: str
:param key: The name of the key to download.
:return: A CommandResult representing the download status.
"""
params = {}
# `download` performs the head_object as well, but the params are
# the same for both operations, so there's nothing missing here.
RequestParamsMapper.map_get_object_params(params, self.params)
with manager:
future = manager.download(fileobj=StdoutBytesWriter(),
bucket=bucket,
key=key,
extra_args=params)
return self._process_transfer(future)
def test_upload_part(self):
params = {}
RequestParamsMapper.map_upload_part_params(params, self.cli_params)
self.assertEqual(params, {
'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key'
})
def test_upload_part(self):
params = {}
RequestParamsMapper.map_upload_part_params(params, self.cli_params)
self.assertEqual(
params,
{'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key'}
)
def __call__(self):
LOGGER.debug("Uploading part copy %s for filename: %s",
self._part_number, self._filename.src)
total_file_size = self._filename.size
start_range = (self._part_number - 1) * self._chunk_size
if self._is_last_part(self._part_number):
end_range = total_file_size - 1
else:
end_range = start_range + self._chunk_size - 1
range_param = 'bytes=%s-%s' % (start_range, end_range)
try:
LOGGER.debug("Waiting for upload id.")
upload_id = self._upload_context.wait_for_upload_id()
bucket, key = find_bucket_key(self._filename.dest)
src_bucket, src_key = find_bucket_key(self._filename.src)
params = {
'Bucket': bucket,
'Key': key,
'PartNumber': self._part_number,
'UploadId': upload_id,
'CopySource': {
'Bucket': src_bucket,
'Key': src_key
},
'CopySourceRange': range_param
}
RequestParamsMapper.map_upload_part_copy_params(
params, self._params)
response_data = self._filename.client.upload_part_copy(**params)
etag = response_data['CopyPartResult']['ETag'][1:-1]
self._upload_context.announce_finished_part(
etag=etag, part_number=self._part_number)
message = print_operation(self._filename, 0)
result = {
'message': message,
'total_parts': self._total_parts(),
'error': False
}
self._result_queue.put(PrintTask(**result))
except UploadCancelledError as e:
# We don't need to do anything in this case. The task
# has been cancelled, and the task that cancelled the
# task has already queued a message.
LOGGER.debug("Not uploading part copy, task has been cancelled.")
except Exception as e:
LOGGER.debug('Error during upload part copy: %s', e, exc_info=True)
message = print_operation(self._filename,
failed=True,
dryrun=False)
message += '\n' + str(e)
result = {'message': message, 'error': True}
self._result_queue.put(PrintTask(**result))
self._upload_context.cancel_upload()
else:
LOGGER.debug("Copy part number %s completed for filename: %s",
self._part_number, self._filename.src)
def set_size_from_s3(self):
"""
This runs a ``HeadObject`` on the s3 object and sets the size.
"""
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket, 'Key': key}
RequestParamsMapper.map_head_object_params(params, self.parameters)
response_data = self.client.head_object(**params)
self.size = int(response_data['ContentLength'])
def create_multipart_upload(self):
bucket, key = find_bucket_key(self.dest)
params = {'Bucket': bucket, 'Key': key}
self._inject_content_type(params)
RequestParamsMapper.map_create_multipart_upload_params(
params, self.parameters)
response_data = self.client.create_multipart_upload(**params)
upload_id = response_data['UploadId']
return upload_id
def create_multipart_upload(self):
bucket, key = find_bucket_key(self.dest)
params = {'Bucket': bucket, 'Key': key}
self._inject_content_type(params)
RequestParamsMapper.map_create_multipart_upload_params(
params, self.parameters)
response_data = self.client.create_multipart_upload(**params)
upload_id = response_data['UploadId']
return upload_id
def test_put_object(self):
params = {}
RequestParamsMapper.map_put_object_params(params, self.cli_params)
self.assertEqual(
params, {
'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'ServerSideEncryption': 'AES256'
})
def _handle_upload(self, body):
bucket, key = find_bucket_key(self.dest)
params = {
'Bucket': bucket,
'Key': key,
'Body': body,
}
self._inject_content_type(params)
RequestParamsMapper.map_put_object_params(params, self.parameters)
response_data = self.client.put_object(**params)
def set_size_from_s3(self):
"""
This runs a ``HeadObject`` on the s3 object and sets the size.
"""
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket,
'Key': key}
RequestParamsMapper.map_head_object_params(params, self.parameters)
response_data = self.client.head_object(**params)
self.size = int(response_data['ContentLength'])
def _handle_upload(self, body):
bucket, key = find_bucket_key(self.dest)
params = {
'Bucket': bucket,
'Key': key,
'Body': body,
}
self._inject_content_type(params)
RequestParamsMapper.map_put_object_params(params, self.parameters)
response_data = self.client.put_object(**params)
def download(self):
"""
Redirects the file to the multipart download function if the file is
large. If it is small enough, it gets the file as an object from s3.
"""
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket, 'Key': key}
RequestParamsMapper.map_get_object_params(params, self.parameters)
response_data = self.client.get_object(**params)
save_file(self.dest, response_data, self.last_update, self.is_stream)
def copy(self):
"""
Copies a object in s3 to another location in s3.
"""
copy_source = self.src
bucket, key = find_bucket_key(self.dest)
params = {'Bucket': bucket, 'CopySource': copy_source, 'Key': key}
self._inject_content_type(params)
RequestParamsMapper.map_copy_object_params(params, self.parameters)
response_data = self.client.copy_object(**params)
def test_put_object(self):
params = {}
RequestParamsMapper.map_put_object_params(params, self.cli_params)
self.assertEqual(
params,
{'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'ServerSideEncryption': 'AES256'}
)
def download(self):
"""
Redirects the file to the multipart download function if the file is
large. If it is small enough, it gets the file as an object from s3.
"""
bucket, key = find_bucket_key(self.src)
params = {'Bucket': bucket, 'Key': key}
RequestParamsMapper.map_get_object_params(params, self.parameters)
response_data = self.client.get_object(**params)
save_file(self.dest, response_data, self.last_update,
self.is_stream)
def test_create_multipart_upload(self):
params = {}
RequestParamsMapper.map_create_multipart_upload_params(
params, self.cli_params)
self.assertEqual(
params,
{'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'ServerSideEncryption': 'AES256'}
)
def copy(self):
"""
Copies a object in s3 to another location in s3.
"""
copy_source = self.src
bucket, key = find_bucket_key(self.dest)
params = {'Bucket': bucket,
'CopySource': copy_source, 'Key': key}
self._inject_content_type(params)
RequestParamsMapper.map_copy_object_params(params, self.parameters)
response_data = self.client.copy_object(**params)
def test_create_multipart_upload(self):
params = {}
RequestParamsMapper.map_create_multipart_upload_params(
params, self.cli_params)
self.assertEqual(
params, {
'SSECustomerAlgorithm': 'AES256',
'SSECustomerKey': 'my-sse-c-key',
'SSEKMSKeyId': 'my-kms-key',
'ServerSideEncryption': 'AES256'
})
def test_put_object(self):
params = {}
RequestParamsMapper.map_put_object_params(params, self.cli_params)
self.assertEqual(
params,
{
"SSECustomerAlgorithm": "AES256",
"SSECustomerKey": "my-sse-c-key",
"SSEKMSKeyId": "my-kms-key",
"ServerSideEncryption": "AES256",
},
)
def test_create_multipart_upload(self):
params = {}
RequestParamsMapper.map_create_multipart_upload_params(params, self.cli_params)
self.assertEqual(
params,
{
"SSECustomerAlgorithm": "AES256",
"SSECustomerKey": "my-sse-c-key",
"SSEKMSKeyId": "my-kms-key",
"ServerSideEncryption": "AES256",
},
)
def test_upload_part_copy(self):
params = {}
RequestParamsMapper.map_upload_part_copy_params(params, self.cli_params)
self.assertEqual(
params,
{
"CopySourceSSECustomerAlgorithm": "AES256",
"CopySourceSSECustomerKey": "my-sse-c-copy-source-key",
"SSECustomerAlgorithm": "AES256",
"SSECustomerKey": "my-sse-c-key",
},
)
def __call__(self):
LOGGER.debug("Uploading part %s for filename: %s", self._part_number,
self._filename.src)
try:
LOGGER.debug("Waiting for upload id.")
upload_id = self._upload_context.wait_for_upload_id()
bucket, key = find_bucket_key(self._filename.dest)
if self._filename.is_stream:
body = self._payload
total = self._upload_context.expected_parts
else:
total = int(
math.ceil(self._filename.size / float(self._chunk_size)))
body = self._read_part()
params = {
'Bucket': bucket,
'Key': key,
'PartNumber': self._part_number,
'UploadId': upload_id,
'Body': body
}
RequestParamsMapper.map_upload_part_params(params, self._params)
try:
response_data = self._filename.client.upload_part(**params)
finally:
body.close()
etag = response_data['ETag'][1:-1]
self._upload_context.announce_finished_part(
etag=etag, part_number=self._part_number)
message = print_operation(self._filename, 0)
result = {'message': message, 'total_parts': total, 'error': False}
self._result_queue.put(PrintTask(**result))
except UploadCancelledError as e:
# We don't need to do anything in this case. The task
# has been cancelled, and the task that cancelled the
# task has already queued a message.
LOGGER.debug("Not uploading part, task has been cancelled.")
except Exception as e:
LOGGER.debug('Error during part upload: %s', e, exc_info=True)
message = print_operation(self._filename,
failed=True,
dryrun=False)
message += '\n' + str(e)
result = {'message': message, 'error': True}
self._result_queue.put(PrintTask(**result))
self._upload_context.cancel_upload()
else:
LOGGER.debug("Part number %s completed for filename: %s",
self._part_number, self._filename.src)
def _upload(self, manager, bucket, key):
"""
Upload stdin using to the specified location.
:type manager: s3transfer.manager.TransferManager
:param manager: The transfer manager to use for the upload.
:type bucket: str
:param bucket: The bucket to upload the stream to.
:type key: str
:param key: The name of the key to upload the stream to.
:return: A CommandResult representing the upload status.
"""
expected_size = self.params.get('expected_size', None)
subscribers = None
if expected_size is not None:
# `expected_size` comes in as a string
expected_size = int(expected_size)
# set the size of the transfer if we know it ahead of time.
subscribers = [ProvideSizeSubscriber(expected_size)]
# TODO: remove when this happens in s3transfer
# If we have the expected size, we can calculate an appropriate
# chunksize based on max parts and chunksize limits
chunksize = find_chunksize(expected_size,
self.config.multipart_chunksize)
else:
# TODO: remove when this happens in s3transfer
# Otherwise, we can still adjust for chunksize limits
chunksize = adjust_chunksize_to_upload_limits(
self.config.multipart_chunksize)
self.config.multipart_chunksize = chunksize
params = {}
RequestParamsMapper.map_put_object_params(params, self.params)
fileobj = NonSeekableStream(binary_stdin)
with manager:
future = manager.upload(fileobj=fileobj,
bucket=bucket,
key=key,
extra_args=params,
subscribers=subscribers)
return self._process_transfer(future)
def __call__(self):
LOGGER.debug("Uploading part %s for filename: %s",
self._part_number, self._filename.src)
try:
LOGGER.debug("Waiting for upload id.")
upload_id = self._upload_context.wait_for_upload_id()
bucket, key = find_bucket_key(self._filename.dest)
if self._filename.is_stream:
body = self._payload
total = self._upload_context.expected_parts
else:
total = int(math.ceil(
self._filename.size/float(self._chunk_size)))
body = self._read_part()
params = {'Bucket': bucket, 'Key': key,
'PartNumber': self._part_number,
'UploadId': upload_id,
'Body': body}
RequestParamsMapper.map_upload_part_params(params, self._params)
try:
response_data = self._filename.client.upload_part(**params)
finally:
body.close()
etag = response_data['ETag'][1:-1]
self._upload_context.announce_finished_part(
etag=etag, part_number=self._part_number)
message = print_operation(self._filename, 0)
result = {'message': message, 'total_parts': total,
'error': False}
self._result_queue.put(PrintTask(**result))
except UploadCancelledError as e:
# We don't need to do anything in this case. The task
# has been cancelled, and the task that cancelled the
# task has already queued a message.
LOGGER.debug("Not uploading part, task has been cancelled.")
except Exception as e:
LOGGER.debug('Error during part upload: %s', e,
exc_info=True)
message = print_operation(self._filename, failed=True,
dryrun=False)
message += '\n' + str(e)
result = {'message': message, 'error': True}
self._result_queue.put(PrintTask(**result))
self._upload_context.cancel_upload()
else:
LOGGER.debug("Part number %s completed for filename: %s",
self._part_number, self._filename.src)
def __call__(self):
LOGGER.debug("Uploading part copy %s for filename: %s",
self._part_number, self._filename.src)
total_file_size = self._filename.size
start_range = (self._part_number - 1) * self._chunk_size
if self._is_last_part(self._part_number):
end_range = total_file_size - 1
else:
end_range = start_range + self._chunk_size - 1
range_param = 'bytes=%s-%s' % (start_range, end_range)
try:
LOGGER.debug("Waiting for upload id.")
upload_id = self._upload_context.wait_for_upload_id()
bucket, key = find_bucket_key(self._filename.dest)
src_bucket, src_key = find_bucket_key(self._filename.src)
params = {'Bucket': bucket, 'Key': key,
'PartNumber': self._part_number,
'UploadId': upload_id,
'CopySource': {'Bucket': src_bucket, 'Key': src_key},
'CopySourceRange': range_param}
RequestParamsMapper.map_upload_part_copy_params(
params, self._params)
response_data = self._filename.client.upload_part_copy(**params)
etag = response_data['CopyPartResult']['ETag'][1:-1]
self._upload_context.announce_finished_part(
etag=etag, part_number=self._part_number)
message = print_operation(self._filename, 0)
result = {'message': message, 'total_parts': self._total_parts(),
'error': False}
self._result_queue.put(PrintTask(**result))
except UploadCancelledError as e:
# We don't need to do anything in this case. The task
# has been cancelled, and the task that cancelled the
# task has already queued a message.
LOGGER.debug("Not uploading part copy, task has been cancelled.")
except Exception as e:
LOGGER.debug('Error during upload part copy: %s', e,
exc_info=True)
message = print_operation(self._filename, failed=True,
dryrun=False)
message += '\n' + str(e)
result = {'message': message, 'error': True}
self._result_queue.put(PrintTask(**result))
self._upload_context.cancel_upload()
else:
LOGGER.debug("Copy part number %s completed for filename: %s",
self._part_number, self._filename.src)
def _download_part(self):
total_file_size = self._filename.size
start_range = self._part_number * self._chunk_size
if self._part_number == int(total_file_size / self._chunk_size) - 1:
end_range = ''
else:
end_range = start_range + self._chunk_size - 1
range_param = 'bytes=%s-%s' % (start_range, end_range)
LOGGER.debug("Downloading bytes range of %s for file %s", range_param,
self._filename.dest)
bucket, key = find_bucket_key(self._filename.src)
params = {'Bucket': bucket, 'Key': key, 'Range': range_param}
RequestParamsMapper.map_get_object_params(params, self._params)
for i in range(self.TOTAL_ATTEMPTS):
try:
LOGGER.debug("Making GetObject requests with byte range: %s",
range_param)
response_data = self._client.get_object(**params)
LOGGER.debug("Response received from GetObject")
body = response_data['Body']
self._queue_writes(body)
self._context.announce_completed_part(self._part_number)
message = print_operation(self._filename, 0)
total_parts = int(self._filename.size / self._chunk_size)
result = {
'message': message,
'error': False,
'total_parts': total_parts
}
self._result_queue.put(PrintTask(**result))
LOGGER.debug("Task complete: %s", self)
return
except (socket.timeout, socket.error, ReadTimeoutError) as e:
LOGGER.debug(
"Timeout error caught, retrying request, "
"(attempt %s / %s)",
i,
self.TOTAL_ATTEMPTS,
exc_info=True)
continue
except IncompleteReadError as e:
LOGGER.debug("Incomplete read detected: %s, (attempt %s / %s)",
e, i, self.TOTAL_ATTEMPTS)
continue
raise RetriesExeededError("Maximum number of attempts exceeded: %s" %
self.TOTAL_ATTEMPTS)
def _download_part(self):
total_file_size = self._filename.size
start_range = self._part_number * self._chunk_size
if self._part_number == int(total_file_size / self._chunk_size) - 1:
end_range = ''
else:
end_range = start_range + self._chunk_size - 1
range_param = 'bytes=%s-%s' % (start_range, end_range)
LOGGER.debug("Downloading bytes range of %s for file %s", range_param,
self._filename.dest)
bucket, key = find_bucket_key(self._filename.src)
params = {'Bucket': bucket,
'Key': key,
'Range': range_param}
RequestParamsMapper.map_get_object_params(params, self._params)
for i in range(self.TOTAL_ATTEMPTS):
try:
LOGGER.debug("Making GetObject requests with byte range: %s",
range_param)
response_data = self._client.get_object(**params)
LOGGER.debug("Response received from GetObject")
body = response_data['Body']
self._queue_writes(body)
self._context.announce_completed_part(self._part_number)
message = print_operation(self._filename, 0)
total_parts = int(self._filename.size / self._chunk_size)
result = {'message': message, 'error': False,
'total_parts': total_parts}
self._result_queue.put(PrintTask(**result))
LOGGER.debug("Task complete: %s", self)
return
except (socket.timeout, socket.error, ReadTimeoutError) as e:
LOGGER.debug("Timeout error caught, retrying request, "
"(attempt %s / %s)", i, self.TOTAL_ATTEMPTS,
exc_info=True)
continue
except IncompleteReadError as e:
LOGGER.debug("Incomplete read detected: %s, (attempt %s / %s)",
e, i, self.TOTAL_ATTEMPTS)
continue
raise RetriesExeededError("Maximum number of attempts exceeded: %s" %
self.TOTAL_ATTEMPTS)
def test_list_objects(self):
params = {}
RequestParamsMapper.map_list_objects_v2_params(params, self.cli_params)
self.assertEqual(params, {'RequestPayer': 'requester'})
def test_upload_part(self):
params = {}
RequestParamsMapper.map_upload_part_params(params, self.cli_params)
self.assertEqual(params, {"SSECustomerAlgorithm": "AES256", "SSECustomerKey": "my-sse-c-key"})
def run(self):
"""
This function wires together all of the generators and completes
the command. First a dictionary is created that is indexed first by
the command name. Then using the instruction, another dictionary
can be indexed to obtain the objects corresponding to the
particular instruction for that command. To begin the wiring,
either a ``FileFormat`` or ``TaskInfo`` object, depending on the
command, is put into a list. Then the function enters a while loop
that pops off an instruction. It then determines the object needed
and calls the call function of the object using the list as the input.
Depending on the number of objects in the input list and the number
of components in the list corresponding to the instruction, the call
method of the component can be called two different ways. If the
number of inputs is equal to the number of components a 1:1 mapping of
inputs to components is used when calling the call function. If the
there are more inputs than components, then a 2:1 mapping of inputs to
components is used where the component call method takes two inputs
instead of one. Whatever files are yielded from the call function
is appended to a list and used as the input for the next repetition
of the while loop until there are no more instructions.
"""
src = self.parameters['src']
dest = self.parameters['dest']
paths_type = self.parameters['paths_type']
files = FileFormat().format(src, dest, self.parameters)
rev_files = FileFormat().format(dest, src, self.parameters)
cmd_translation = {}
cmd_translation['locals3'] = {
'cp': 'upload',
'sync': 'upload',
'mv': 'move'
}
cmd_translation['s3s3'] = {'cp': 'copy', 'sync': 'copy', 'mv': 'move'}
cmd_translation['s3local'] = {
'cp': 'download',
'sync': 'download',
'mv': 'move'
}
cmd_translation['s3'] = {
'rm': 'delete',
'mb': 'make_bucket',
'rb': 'remove_bucket'
}
result_queue = queue.Queue()
operation_name = cmd_translation[paths_type][self.cmd]
fgen_kwargs = {
'client': self._source_client,
'operation_name': operation_name,
'follow_symlinks': self.parameters['follow_symlinks'],
'page_size': self.parameters['page_size'],
'result_queue': result_queue
}
rgen_kwargs = {
'client': self._client,
'operation_name': '',
'follow_symlinks': self.parameters['follow_symlinks'],
'page_size': self.parameters['page_size'],
'result_queue': result_queue
}
fgen_request_parameters = {}
fgen_head_object_params = {}
fgen_request_parameters['HeadObject'] = fgen_head_object_params
fgen_kwargs['request_parameters'] = fgen_request_parameters
# SSE-C may be neaded for HeadObject for copies/downloads/deletes
# If the operation is s3 to s3, the FileGenerator should use the
# copy source key and algorithm. Otherwise, use the regular
# SSE-C key and algorithm. Note the reverse FileGenerator does
# not need any of these because it is used only for sync operations
# which only use ListObjects which does not require HeadObject.
RequestParamsMapper.map_head_object_params(fgen_head_object_params,
self.parameters)
if paths_type == 's3s3':
RequestParamsMapper.map_head_object_params(
fgen_head_object_params, {
'sse_c': self.parameters.get('sse_c_copy_source'),
'sse_c_key': self.parameters.get('sse_c_copy_source_key')
})
file_generator = FileGenerator(**fgen_kwargs)
rev_generator = FileGenerator(**rgen_kwargs)
taskinfo = [
TaskInfo(src=files['src']['path'],
src_type='s3',
operation_name=operation_name,
client=self._client)
]
stream_dest_path, stream_compare_key = find_dest_path_comp_key(files)
stream_file_info = [
FileInfo(src=files['src']['path'],
dest=stream_dest_path,
compare_key=stream_compare_key,
src_type=files['src']['type'],
dest_type=files['dest']['type'],
operation_name=operation_name,
client=self._client,
is_stream=True)
]
file_info_builder = FileInfoBuilder(self._client, self._source_client,
self.parameters)
s3handler = S3Handler(self.session,
self.parameters,
runtime_config=self._runtime_config,
result_queue=result_queue)
s3_stream_handler = S3StreamHandler(self.session,
self.parameters,
result_queue=result_queue)
sync_strategies = self.choose_sync_strategies()
command_dict = {}
if self.cmd == 'sync':
command_dict = {
'setup': [files, rev_files],
'file_generator': [file_generator, rev_generator],
'filters': [
create_filter(self.parameters),
create_filter(self.parameters)
],
'comparator': [Comparator(**sync_strategies)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'cp' and self.parameters['is_stream']:
command_dict = {
'setup': [stream_file_info],
's3_handler': [s3_stream_handler]
}
elif self.cmd == 'cp':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'rm':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'mv':
command_dict = {
'setup': [files],
'file_generator': [file_generator],
'filters': [create_filter(self.parameters)],
'file_info_builder': [file_info_builder],
's3_handler': [s3handler]
}
elif self.cmd == 'mb':
command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
elif self.cmd == 'rb':
command_dict = {'setup': [taskinfo], 's3_handler': [s3handler]}
files = command_dict['setup']
while self.instructions:
instruction = self.instructions.pop(0)
file_list = []
components = command_dict[instruction]
for i in range(len(components)):
if len(files) > len(components):
file_list.append(components[i].call(*files))
else:
file_list.append(components[i].call(files[i]))
files = file_list
# This is kinda quirky, but each call through the instructions
# will replaces the files attr with the return value of the
# file_list. The very last call is a single list of
# [s3_handler], and the s3_handler returns the number of
# tasks failed and the number of tasks warned.
# This means that files[0] now contains a namedtuple with
# the number of failed tasks and the number of warned tasks.
# In terms of the RC, we're keeping it simple and saying
# that > 0 failed tasks will give a 1 RC and > 0 warned
# tasks will give a 2 RC. Otherwise a RC of zero is returned.
rc = 0
if files[0].num_tasks_failed > 0:
rc = 1
if files[0].num_tasks_warned > 0:
rc = 2
return rc
def test_upload_part_copy(self):
params = {}
RequestParamsMapper.map_upload_part_copy_params(
params, self.cli_params)
self.assertEqual(params, {'RequestPayer': 'requester'})
def test_upload_part_copy(self):
params = {}
RequestParamsMapper.map_upload_part_copy_params(
params, self.cli_params)
self.assertEqual(params, {'RequestPayer': 'requester'})
def test_delete_object(self):
params = {}
RequestParamsMapper.map_delete_object_params(
params, self.cli_params)
self.assertEqual(params, {'RequestPayer': 'requester'})
def test_list_objects(self):
params = {}
RequestParamsMapper.map_list_objects_v2_params(
params, self.cli_params)
self.assertEqual(params, {'RequestPayer': 'requester'})
def test_delete_object(self):
params = {}
RequestParamsMapper.map_delete_object_params(params, self.cli_params)
self.assertEqual(params, {'RequestPayer': 'requester'})
def run(self):
"""
This function wires together all of the generators and completes
the command. First a dictionary is created that is indexed first by
the command name. Then using the instruction, another dictionary
can be indexed to obtain the objects corresponding to the
particular instruction for that command. To begin the wiring,
either a ``FileFormat`` or ``TaskInfo`` object, depending on the
command, is put into a list. Then the function enters a while loop
that pops off an instruction. It then determines the object needed
and calls the call function of the object using the list as the input.
Depending on the number of objects in the input list and the number
of components in the list corresponding to the instruction, the call
method of the component can be called two different ways. If the
number of inputs is equal to the number of components a 1:1 mapping of
inputs to components is used when calling the call function. If the
there are more inputs than components, then a 2:1 mapping of inputs to
components is used where the component call method takes two inputs
instead of one. Whatever files are yielded from the call function
is appended to a list and used as the input for the next repetition
of the while loop until there are no more instructions.
"""
src = self.parameters["src"]
dest = self.parameters["dest"]
paths_type = self.parameters["paths_type"]
files = FileFormat().format(src, dest, self.parameters)
rev_files = FileFormat().format(dest, src, self.parameters)
cmd_translation = {}
cmd_translation["locals3"] = {"cp": "upload", "sync": "upload", "mv": "move"}
cmd_translation["s3s3"] = {"cp": "copy", "sync": "copy", "mv": "move"}
cmd_translation["s3local"] = {"cp": "download", "sync": "download", "mv": "move"}
cmd_translation["s3"] = {"rm": "delete", "mb": "make_bucket", "rb": "remove_bucket"}
result_queue = queue.Queue()
operation_name = cmd_translation[paths_type][self.cmd]
fgen_kwargs = {
"client": self._source_client,
"operation_name": operation_name,
"follow_symlinks": self.parameters["follow_symlinks"],
"page_size": self.parameters["page_size"],
"result_queue": result_queue,
}
rgen_kwargs = {
"client": self._client,
"operation_name": "",
"follow_symlinks": self.parameters["follow_symlinks"],
"page_size": self.parameters["page_size"],
"result_queue": result_queue,
}
fgen_request_parameters = {}
fgen_head_object_params = {}
fgen_request_parameters["HeadObject"] = fgen_head_object_params
fgen_kwargs["request_parameters"] = fgen_request_parameters
# SSE-C may be neaded for HeadObject for copies/downloads/deletes
# If the operation is s3 to s3, the FileGenerator should use the
# copy source key and algorithm. Otherwise, use the regular
# SSE-C key and algorithm. Note the reverse FileGenerator does
# not need any of these because it is used only for sync operations
# which only use ListObjects which does not require HeadObject.
RequestParamsMapper.map_head_object_params(fgen_head_object_params, self.parameters)
if paths_type == "s3s3":
RequestParamsMapper.map_head_object_params(
fgen_head_object_params,
{
"sse_c": self.parameters.get("sse_c_copy_source"),
"sse_c_key": self.parameters.get("sse_c_copy_source_key"),
},
)
file_generator = FileGenerator(**fgen_kwargs)
rev_generator = FileGenerator(**rgen_kwargs)
taskinfo = [
TaskInfo(src=files["src"]["path"], src_type="s3", operation_name=operation_name, client=self._client)
]
stream_dest_path, stream_compare_key = find_dest_path_comp_key(files)
stream_file_info = [
FileInfo(
src=files["src"]["path"],
dest=stream_dest_path,
compare_key=stream_compare_key,
src_type=files["src"]["type"],
dest_type=files["dest"]["type"],
operation_name=operation_name,
client=self._client,
is_stream=True,
)
]
file_info_builder = FileInfoBuilder(self._client, self._source_client, self.parameters)
s3handler = S3Handler(
self.session, self.parameters, runtime_config=self._runtime_config, result_queue=result_queue
)
s3_stream_handler = S3StreamHandler(self.session, self.parameters, result_queue=result_queue)
sync_strategies = self.choose_sync_strategies()
command_dict = {}
if self.cmd == "sync":
command_dict = {
"setup": [files, rev_files],
"file_generator": [file_generator, rev_generator],
"filters": [create_filter(self.parameters), create_filter(self.parameters)],
"comparator": [Comparator(**sync_strategies)],
"file_info_builder": [file_info_builder],
"s3_handler": [s3handler],
}
elif self.cmd == "cp" and self.parameters["is_stream"]:
command_dict = {"setup": [stream_file_info], "s3_handler": [s3_stream_handler]}
elif self.cmd == "cp":
command_dict = {
"setup": [files],
"file_generator": [file_generator],
"filters": [create_filter(self.parameters)],
"file_info_builder": [file_info_builder],
"s3_handler": [s3handler],
}
elif self.cmd == "rm":
command_dict = {
"setup": [files],
"file_generator": [file_generator],
"filters": [create_filter(self.parameters)],
"file_info_builder": [file_info_builder],
"s3_handler": [s3handler],
}
elif self.cmd == "mv":
command_dict = {
"setup": [files],
"file_generator": [file_generator],
"filters": [create_filter(self.parameters)],
"file_info_builder": [file_info_builder],
"s3_handler": [s3handler],
}
elif self.cmd == "mb":
command_dict = {"setup": [taskinfo], "s3_handler": [s3handler]}
elif self.cmd == "rb":
command_dict = {"setup": [taskinfo], "s3_handler": [s3handler]}
files = command_dict["setup"]
while self.instructions:
instruction = self.instructions.pop(0)
file_list = []
components = command_dict[instruction]
for i in range(len(components)):
if len(files) > len(components):
file_list.append(components[i].call(*files))
else:
file_list.append(components[i].call(files[i]))
files = file_list
# This is kinda quirky, but each call through the instructions
# will replaces the files attr with the return value of the
# file_list. The very last call is a single list of
# [s3_handler], and the s3_handler returns the number of
# tasks failed and the number of tasks warned.
# This means that files[0] now contains a namedtuple with
# the number of failed tasks and the number of warned tasks.
# In terms of the RC, we're keeping it simple and saying
# that > 0 failed tasks will give a 1 RC and > 0 warned
# tasks will give a 2 RC. Otherwise a RC of zero is returned.
rc = 0
if files[0].num_tasks_failed > 0:
rc = 1
if files[0].num_tasks_warned > 0:
rc = 2
return rc
