def _update_rate_limit(self, transferred_size): from nimbus.offsite import queue_service if self.update_rate_limit_time > 0: if self._last_update_rate_limit is None: self._last_update_rate_limit = time() else: now = time() diff = now - self._last_update_rate_limit if diff > self.update_rate_limit_time: self.rate_limit = queue_service.get_worker_ratelimit() self.start = time() self._transferred_size = transferred_size self._last_update_rate_limit = now
def _upload(self, filename, keyname): from nimbus.offsite import queue_service self.logger.info('simple upload') key = self.bucket.new_key(keyname) self.rate_limiter.reset( queue_service.get_worker_ratelimit() ) md5 = utils.md5_for_large_file(filename) headers={'x-amz-meta-nimbus-md5' : md5} with file(filename) as f_obj: key.set_contents_from_file(f_obj, cb=self.callbacks, headers=headers, num_cb=-1)
def _upload_multipart(self, filename, keyname, part): from nimbus.offsite import queue_service self.logger.info('initiate multipart upload ') md5 = utils.md5_for_large_file(filename) headers={'x-amz-meta-nimbus-md5' : md5} multipart = self.bucket.initiate_multipart_upload(keyname, headers=headers) self.logger.info('initiate multipart with part = %d' % (part) ) with MultipartFileManager(filename, part) as manager: for (part_number, part_content) in enumerate(manager, part): self.rate_limiter.reset( queue_service.get_worker_ratelimit() ) # FIX THIS: quantidade transferida e total sempre #eh reportada como o da parte part_file = StringIO(part_content) multipart.upload_part_from_file(part_file, part_number + 1, cb=self.callbacks, num_cb=-1) self.multipart_status_callbacks(filename, part_number) size = os.path.getsize(filename) sent_size = (part_number * MIN_MULTIPART_SIZE) + len(part_content) if size == sent_size: multipart.complete_upload() else: raise IncompleteUpload("Not sent full file size")