def setUp(self):
self.monitor = TransferMonitor()
self.transfer_id = self.monitor.notify_new_transfer()
self.meta = ProcessPoolTransferMeta(transfer_id=self.transfer_id,
call_args=CallArgs())
self.future = ProcessPoolTransferFuture(monitor=self.monitor,
meta=self.meta)
def setUp(self):
super(TestUploadFilenameInputManager, self).setUp()
self.upload_input_manager = UploadFilenameInputManager(
self.osutil, self.transfer_coordinator)
self.call_args = CallArgs(fileobj=self.filename,
subscribers=self.subscribers)
self.future = self.get_transfer_future(self.call_args)
def setUp(self):
self.subscriber = RecordingSubscriber()
self.second_subscriber = RecordingSubscriber()
self.call_args = CallArgs(
subscribers=[self.subscriber, self.second_subscriber])
self.transfer_meta = TransferMeta(self.call_args)
self.transfer_future = TransferFuture(self.transfer_meta)
def delete(self, bucket, key, extra_args=None, subscribers=None):
"""Delete an S3 object.
:type bucket: str
:param bucket: The name of the bucket.
:type key: str
:param key: The name of the S3 object to delete.
:type extra_args: dict
:param extra_args: Extra arguments that may be passed to the
DeleteObject call.
:type subscribers: list
:param subscribers: A list of subscribers to be invoked during the
process of the transfer request. Note that the ``on_progress``
callback is not invoked during object deletion.
:rtype: s3transfer.futures.TransferFuture
:return: Transfer future representing the deletion.
"""
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = []
self._validate_all_known_args(extra_args, self.ALLOWED_DELETE_ARGS)
call_args = CallArgs(bucket=bucket,
key=key,
extra_args=extra_args,
subscribers=subscribers)
return self._submit_transfer(call_args, DeleteSubmissionTask)
def get_call_args(self, **kwargs):
default_call_args = {
'fileobj': self.filename, 'bucket': self.bucket,
'key': self.key, 'extra_args': self.extra_args,
'subscribers': self.subscribers
}
default_call_args.update(kwargs)
return CallArgs(**default_call_args)
def setUp(self):
super(TestUploadNonSeekableInputManager, self).setUp()
self.upload_input_manager = UploadNonSeekableInputManager(
self.osutil, self.transfer_coordinator)
self.fileobj = NonSeekableReader(self.content)
self.call_args = CallArgs(fileobj=self.fileobj,
subscribers=self.subscribers)
self.future = self.get_transfer_future(self.call_args)
def copy(self,
copy_source,
bucket,
key,
extra_args=None,
subscribers=None,
source_client=None):
"""Copies a file in S3
:type copy_source: dict
:param copy_source: The name of the source bucket, key name of the
source object, and optional version ID of the source object. The
dictionary format is:
``{'Bucket': 'bucket', 'Key': 'key', 'VersionId': 'id'}``. Note
that the ``VersionId`` key is optional and may be omitted.
:type bucket: str
:param bucket: The name of the bucket to copy to
:type key: str
:param key: The name of the key to copy to
:type extra_args: dict
:param extra_args: Extra arguments that may be passed to the
client operation
:type subscribers: a list of subscribers
:param subscribers: The list of subscribers to be invoked in the
order provided based on the event emit during the process of
the transfer request.
:type source_client: botocore or boto3 Client
:param source_client: The client to be used for operation that
may happen at the source object. For example, this client is
used for the head_object that determines the size of the copy.
If no client is provided, the transfer manager's client is used
as the client for the source object.
:rtype: s3transfer.futures.TransferFuture
:returns: Transfer future representing the copy
"""
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = []
if source_client is None:
source_client = self._client
self._validate_all_known_args(extra_args, self.ALLOWED_COPY_ARGS)
if isinstance(copy_source, dict):
self._validate_if_bucket_supported(copy_source.get('Bucket'))
self._validate_if_bucket_supported(bucket)
call_args = CallArgs(copy_source=copy_source,
bucket=bucket,
key=key,
extra_args=extra_args,
subscribers=subscribers,
source_client=source_client)
return self._submit_transfer(call_args, CopySubmissionTask)
def download(self, bucket, key, fileobj, extra_args=None,
subscribers=None):
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = {}
callargs = CallArgs(bucket=bucket, key=key, fileobj=fileobj,
extra_args=extra_args, subscribers=subscribers)
return self._submit_transfer("get_object", callargs)
def delete(self, bucket, key, extra_args=None,
subscribers=None):
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = {}
callargs = CallArgs(
bucket=bucket, key=key, extra_args=extra_args,
subscribers=subscribers)
return self._submit_transfer("delete_object", callargs)
def setUp(self):
super(TestDownloadSeekableOutputManager, self).setUp()
self.download_output_manager = DownloadSeekableOutputManager(
self.osutil, self.transfer_coordinator,
io_executor=self.io_executor)
# Create a fileobj to write to
self.fileobj = open(self.filename, 'wb')
self.call_args = CallArgs(fileobj=self.fileobj)
self.future = self.get_transfer_future(self.call_args)
def setUp(self):
super(BaseDownloadOutputManagerTest, self).setUp()
self.osutil = OSUtils()
# Create a file to write to
self.tempdir = tempfile.mkdtemp()
self.filename = os.path.join(self.tempdir, 'myfile')
self.call_args = CallArgs(fileobj=self.filename)
self.future = self.get_transfer_future(self.call_args)
self.io_executor = BoundedExecutor(1000, 1)
def download_file(self,
bucket,
key,
filename,
extra_args=None,
expected_size=None):
"""Downloads the object's contents to a file
:type bucket: str
:param bucket: The name of the bucket to download from
:type key: str
:param key: The name of the key to download from
:type filename: str
:param filename: The name of a file to download to.
:type extra_args: dict
:param extra_args: Extra arguments that may be passed to the
client operation
:type expected_size: int
:param expected_size: The expected size in bytes of the download. If
provided, the downloader will not call HeadObject to determine the
object's size and use the provided value instead. The size is
needed to determine whether to do a multipart download.
:rtype: s3transfer.futures.TransferFuture
:returns: Transfer future representing the download
"""
self._start_if_needed()
if extra_args is None:
extra_args = {}
self._validate_all_known_args(extra_args)
transfer_id = self._transfer_monitor.notify_new_transfer()
download_file_request = DownloadFileRequest(
transfer_id=transfer_id,
bucket=bucket,
key=key,
filename=filename,
extra_args=extra_args,
expected_size=expected_size,
)
logger.debug('Submitting download file request: %s.',
download_file_request)
self._download_request_queue.put(download_file_request)
call_args = CallArgs(bucket=bucket,
key=key,
filename=filename,
extra_args=extra_args,
expected_size=expected_size)
future = self._get_transfer_future(transfer_id, call_args)
return future
def test_delete_request(self):
callargs = CallArgs(bucket=self.bucket,
key=self.key,
extra_args={},
subscribers=[])
coordinator = s3transfer.crt.CRTTransferCoordinator()
future = s3transfer.crt.CRTTransferFuture(
s3transfer.crt.CRTTransferMeta(call_args=callargs), coordinator)
crt_request = self.request_serializer.serialize_http_request(
"delete_object", future)
self.assertEqual("DELETE", crt_request.method)
self.assertEqual(self.expected_path, crt_request.path)
self.assertEqual(self.expected_host, crt_request.headers.get("host"))
self.assertIsNone(crt_request.headers.get("Authorization"))
def download(self,
bucket,
key,
fileobj,
extra_args=None,
subscribers=None):
"""Downloads a file from S3
:type bucket: str
:param bucket: The name of the bucket to download from
:type key: str
:param key: The name of the key to download from
:type fileobj: str or seekable file-like object
:param fileobj: The name of a file to download or a seekable file-like
object to download. It is recommended to use a filename because
file-like objects may result in higher memory usage.
:type extra_args: dict
:param extra_args: Extra arguments that may be passed to the
client operation
:type subscribers: list(s3transfer.subscribers.BaseSubscriber)
:param subscribers: The list of subscribers to be invoked in the
order provided based on the event emit during the process of
the transfer request.
:rtype: s3transfer.futures.TransferFuture
:returns: Transfer future representing the download
"""
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = []
self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)
self._validate_if_bucket_supported(bucket)
call_args = CallArgs(bucket=bucket,
key=key,
fileobj=fileobj,
extra_args=extra_args,
subscribers=subscribers)
extra_main_kwargs = {'io_executor': self._io_executor}
if self._bandwidth_limiter:
extra_main_kwargs['bandwidth_limiter'] = self._bandwidth_limiter
return self._submit_transfer(call_args, DownloadSubmissionTask,
extra_main_kwargs)
def download(self,
bucket,
key,
fileobj,
extra_args=None,
subscribers=None):
"""Downloads a file from S3
:type bucket: str
:param bucket: The name of the bucket to download from
:type key: str
:param key: The name of the key to download from
:type fileobj: str
:param fileobj: The name of a file to download to.
:type extra_args: dict
:param extra_args: Extra arguments that may be passed to the
client operation
:type subscribers: list(s3transfer.subscribers.BaseSubscriber)
:param subscribers: The list of subscribers to be invoked in the
order provided based on the event emit during the process of
the transfer request.
:rtype: s3transfer.futures.TransferFuture
:returns: Transfer future representing the download
"""
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = []
self._validate_all_known_args(extra_args, self.ALLOWED_DOWNLOAD_ARGS)
call_args = CallArgs(bucket=bucket,
key=key,
fileobj=fileobj,
extra_args=extra_args,
subscribers=subscribers)
return self._submit_transfer(call_args, DownloadSubmissionTask,
{'io_executor': self._io_executor})
def upload(self, fileobj, bucket, key, extra_args=None, subscribers=None):
"""Uploads a file to S3
:type fileobj: str or seekable file-like object
:param fileobj: The name of a file to upload or a seekable file-like
object to upload. It is recommended to use a filename because
file-like objects may result in higher memory usage.
:type bucket: str
:param bucket: The name of the bucket to upload to
:type key: str
:param key: The name of the key to upload to
:type extra_args: dict
:param extra_args: Extra arguments that may be passed to the
client operation
:type subscribers: list(s3transfer.subscribers.BaseSubscriber)
:param subscribers: The list of subscribers to be invoked in the
order provided based on the event emit during the process of
the transfer request.
:rtype: s3transfer.futures.TransferFuture
:returns: Transfer future representing the upload
"""
if extra_args is None:
extra_args = {}
if subscribers is None:
subscribers = []
self._validate_all_known_args(extra_args, self.ALLOWED_UPLOAD_ARGS)
call_args = CallArgs(fileobj=fileobj,
bucket=bucket,
key=key,
extra_args=extra_args,
subscribers=subscribers)
return self._submit_transfer(call_args, UploadSubmissionTask)
def test_call_args(self):
call_args = CallArgs(foo='bar', biz='baz')
self.assertEqual(call_args.foo, 'bar')
self.assertEqual(call_args.biz, 'baz')
def test_transfer_id(self):
meta = ProcessPoolTransferMeta(1, CallArgs())
self.assertEqual(meta.transfer_id, 1)
def test_call_args(self):
call_args = CallArgs()
meta = ProcessPoolTransferMeta(1, call_args)
self.assertEqual(meta.call_args, call_args)
def setUp(self):
super(TestSubmissionTask, self).setUp()
self.executor = BoundedExecutor(1000, 5)
self.call_args = CallArgs(subscribers=[])
self.transfer_future = self.get_transfer_future(self.call_args)
self.main_kwargs = {'transfer_future': self.transfer_future}
def test_user_context(self):
meta = ProcessPoolTransferMeta(1, CallArgs())
self.assertEqual(meta.user_context, {})
meta.user_context['mykey'] = 'myvalue'
self.assertEqual(meta.user_context, {'mykey': 'myvalue'})
