def test_open_file_chunk_reader(self): reader = OSUtils().open_file_chunk_reader(self.filename, 0, 3, [self.callback]) # The returned reader should be a ReadFileChunk. self.assertIsInstance(reader, ReadFileChunk) # The content of the reader should be correct. self.assertEqual(reader.read(), self.content) # Callbacks should be disabled depspite being passed in. self.assertEqual(self.amounts_seen, [])
def test_open_file_chunk_reader_from_fileobj(self): with open(self.filename, 'rb') as f: reader = OSUtils().open_file_chunk_reader_from_fileobj( f, len(self.content), len(self.content), [self.callback]) # The returned reader should be a ReadFileChunk. self.assertIsInstance(reader, ReadFileChunk) # The content of the reader should be correct. self.assertEqual(reader.read(), self.content) reader.close() # Callbacks should be disabled depspite being passed in. self.assertEqual(self.amounts_seen, []) self.assertEqual(self.num_close_callback_calls, 0)
def setUp(self): super(TestGetObjectWorker, self).setUp() self.files = FileCreator() self.queue = queue.Queue() self.client_factory = mock.Mock(ClientFactory) self.client_factory.create_client.return_value = self.client self.transfer_monitor = TransferMonitor() self.osutil = OSUtils() self.worker = GetObjectWorker( queue=self.queue, client_factory=self.client_factory, transfer_monitor=self.transfer_monitor, osutil=self.osutil ) self.transfer_id = self.transfer_monitor.notify_new_transfer() self.bucket = 'bucket' self.key = 'key' self.remote_contents = b'my content' self.temp_filename = self.files.create_file('tempfile', '') self.extra_args = {} self.offset = 0 self.final_filename = self.files.full_path('final_filename') self.stream = six.BytesIO(self.remote_contents) self.transfer_monitor.notify_expected_jobs_to_complete( self.transfer_id, 1000)
def __init__(self, client, config=None, osutil=None, executor_cls=None): """A transfer manager interface for Amazon S3 :param client: Client to be used by the manager :param config: TransferConfig to associate specific configurations :param osutil: OSUtils object to use for os-related behavior when using with transfer manager. :type executor_cls: ibm_s3transfer.futures.BaseExecutor :param executor_cls: The class of executor to use with the transfer manager. By default, concurrent.futures.ThreadPoolExecutor is used. """ self._client = client self._config = config if config is None: self._config = TransferConfig() self._osutil = osutil if osutil is None: self._osutil = OSUtils() self._coordinator_controller = TransferCoordinatorController() # A counter to create unique id's for each transfer submitted. self._id_counter = 0 # The executor responsible for making S3 API transfer requests self._request_executor = BoundedExecutor( max_size=self._config.max_request_queue_size, max_num_threads=self._config.max_request_concurrency, tag_semaphores={ IN_MEMORY_UPLOAD_TAG: TaskSemaphore(self._config.max_in_memory_upload_chunks), IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore( self._config.max_in_memory_download_chunks) }, executor_cls=executor_cls) # The executor responsible for submitting the necessary tasks to # perform the desired transfer self._submission_executor = BoundedExecutor( max_size=self._config.max_submission_queue_size, max_num_threads=self._config.max_submission_concurrency, executor_cls=executor_cls) # There is one thread available for writing to disk. It will handle # downloads for all files. self._io_executor = BoundedExecutor( max_size=self._config.max_io_queue_size, max_num_threads=1, executor_cls=executor_cls) # The component responsible for limiting bandwidth usage if it # is configured. self._bandwidth_limiter = None if self._config.max_bandwidth is not None: logger.debug('Setting max_bandwidth to %s', self._config.max_bandwidth) leaky_bucket = LeakyBucket(self._config.max_bandwidth) self._bandwidth_limiter = BandwidthLimiter(leaky_bucket) self._register_handlers()
def __init__(self, client_kwargs=None, config=None): """Downloads S3 objects using process pools :type client_kwargs: dict :param client_kwargs: The keyword arguments to provide when instantiating S3 clients. The arguments must match the keyword arguments provided to the `ibm_botocore.session.Session.create_client()` method. :type config: ProcessTransferConfig :param config: Configuration for the downloader """ if client_kwargs is None: client_kwargs = {} self._client_factory = ClientFactory(client_kwargs) self._transfer_config = config if config is None: self._transfer_config = ProcessTransferConfig() self._download_request_queue = multiprocessing.Queue(1000) self._worker_queue = multiprocessing.Queue(1000) self._osutil = OSUtils() self._started = False self._start_lock = threading.Lock() # These below are initialized in the start() method self._manager = None self._transfer_monitor = None self._submitter = None self._workers = []
def test_get_temp_filename(self): filename = 'myfile' self.assertIsNotNone( re.match( '%s\.[0-9A-Fa-f]{8}$' % filename, OSUtils().get_temp_filename(filename) ) )
def test_remove_file_ignores_errors(self): non_existent_file = os.path.join(self.tempdir, 'no-exist') # This should not exist to start. self.assertFalse(os.path.exists(non_existent_file)) try: OSUtils().remove_file(non_existent_file) except OSError as e: self.fail('OSError should have been caught: %s' % e)
def setUp(self): super(BaseSubmissionTaskTest, self).setUp() self.config = TransferConfig() self.osutil = OSUtils() self.executor = BoundedExecutor( 1000, 1, { IN_MEMORY_UPLOAD_TAG: TaskSemaphore(10), IN_MEMORY_DOWNLOAD_TAG: SlidingWindowSemaphore(10) })
def setUp(self): super(BaseDownloadOutputManagerTest, self).setUp() self.osutil = OSUtils() # Create a file to write to self.tempdir = tempfile.mkdtemp() self.filename = os.path.join(self.tempdir, 'myfile') self.call_args = CallArgs(fileobj=self.filename) self.future = self.get_transfer_future(self.call_args) self.io_executor = BoundedExecutor(1000, 1)
def setUp(self): super(TestGetObjectTask, self).setUp() self.bucket = 'mybucket' self.key = 'mykey' self.extra_args = {} self.callbacks = [] self.max_attempts = 5 self.io_executor = BoundedExecutor(1000, 1) self.content = b'my content' self.stream = six.BytesIO(self.content) self.fileobj = WriteCollector() self.osutil = OSUtils() self.io_chunksize = 64 * (1024**2) self.task_cls = GetObjectTask self.download_output_manager = DownloadSeekableOutputManager( self.osutil, self.transfer_coordinator, self.io_executor)
def __init__(self, client=None, config=None, osutil=None, manager=None): if not client and not manager: raise ValueError( 'Either a ibm_boto3.Client or ibm_s3transfer.manager.TransferManager ' 'must be provided') if manager and any([client, config, osutil]): raise ValueError( 'Manager cannot be provided with client, config, ' 'nor osutil. These parameters are mutually exclusive.') if config is None: config = TransferConfig() if osutil is None: osutil = OSUtils() if manager: self._manager = manager else: self._manager = create_transfer_manager(client, config, osutil)
def setUp(self): super(BaseUploadTest, self).setUp() self.bucket = 'mybucket' self.key = 'foo' self.osutil = OSUtils() self.tempdir = tempfile.mkdtemp() self.filename = os.path.join(self.tempdir, 'myfile') self.content = b'my content' self.subscribers = [] with open(self.filename, 'wb') as f: f.write(self.content) # A list to keep track of all of the bodies sent over the wire # and their order. self.sent_bodies = [] self.client.meta.events.register('before-parameter-build.s3.*', self.collect_body)
def test_is_special_file_for_normal_file(self): self.assertFalse(OSUtils().is_special_file(self.filename))
def test_rename_file(self): new_filename = os.path.join(self.tempdir, 'newfoo') OSUtils().rename_file(self.filename, new_filename) self.assertFalse(os.path.exists(self.filename)) self.assertTrue(os.path.exists(new_filename))
def test_remove_file_proxies_remove_file(self): OSUtils().remove_file(self.filename) self.assertFalse(os.path.exists(self.filename))
def test_character_device(self): self.assertTrue(OSUtils().is_special_file('/dev/null'))
def test_open_file(self): fileobj = OSUtils().open(os.path.join(self.tempdir, 'foo'), 'w') self.assertTrue(hasattr(fileobj, 'write'))
def test_fifo(self): os.mkfifo(self.filename) self.assertTrue(OSUtils().is_special_file(self.filename))
def setUp(self): super(BaseIOTaskTest, self).setUp() self.files = FileCreator() self.osutil = OSUtils() self.temp_filename = os.path.join(self.files.rootdir, 'mytempfile') self.final_filename = os.path.join(self.files.rootdir, 'myfile')
def test_get_file_size(self): self.assertEqual(OSUtils().get_file_size(self.filename), len(self.content))
def test_allocate(self): truncate_size = 1 OSUtils().allocate(self.filename, truncate_size) with open(self.filename, 'rb') as f: self.assertEqual(len(f.read()), truncate_size)
def test_is_not_compatible_for_non_special_file(self): self.assertFalse( self.download_output_manager.is_compatible(self.filename, OSUtils()))
def test_is_special_file_for_non_existant_file(self): non_existant_filename = os.path.join(self.tempdir, 'no-exist') self.assertFalse(os.path.exists(non_existant_filename)) self.assertFalse(OSUtils().is_special_file(non_existant_filename))
def test_get_temp_filename_len_gt_255(self): filename = 'a' * 280 temp_filename = OSUtils().get_temp_filename(filename) self.assertLessEqual(len(temp_filename), 255)
def test_socket(self): sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) sock.bind(self.filename) self.assertTrue(OSUtils().is_special_file(self.filename))
def test_allocate_with_os_error(self, mock_fallocate): mock_fallocate.side_effect = OSError() with self.assertRaises(OSError): OSUtils().allocate(self.filename, 1) self.assertFalse(os.path.exists(self.filename))
def setUp(self): super(BaseUploadInputManagerTest, self).setUp() self.osutil = OSUtils() self.config = TransferConfig() self.recording_subscriber = RecordingSubscriber() self.subscribers.append(self.recording_subscriber)