Example #1
0
File: S3.py Project: veezor/Nimbus
 def _update_rate_limit(self, transferred_size):
     from nimbus.offsite import queue_service
     if self.update_rate_limit_time > 0:
         if self._last_update_rate_limit is None:
             self._last_update_rate_limit = time()
         else:
             now = time()
             diff = now - self._last_update_rate_limit
             if diff > self.update_rate_limit_time:
                 self.rate_limit = queue_service.get_worker_ratelimit()
                 self.start = time()
                 self._transferred_size = transferred_size
                 self._last_update_rate_limit = now
Example #2
0
File: S3.py Project: veezor/Nimbus
    def _upload(self, filename, keyname):
        from nimbus.offsite import queue_service
        self.logger.info('simple upload')
        key = self.bucket.new_key(keyname)
        self.rate_limiter.reset( queue_service.get_worker_ratelimit() )
        md5 = utils.md5_for_large_file(filename)
        headers={'x-amz-meta-nimbus-md5' : md5}

        with file(filename) as f_obj:
            key.set_contents_from_file(f_obj,
                                       cb=self.callbacks,
                                       headers=headers,
                                       num_cb=-1)
Example #3
0
File: S3.py Project: veezor/Nimbus
    def _upload_multipart(self, filename, keyname, part):
        from nimbus.offsite import queue_service

        self.logger.info('initiate multipart upload ')


        md5 = utils.md5_for_large_file(filename)
        headers={'x-amz-meta-nimbus-md5' : md5}
        multipart = self.bucket.initiate_multipart_upload(keyname, headers=headers)

        self.logger.info('initiate multipart with part =  %d' % (part)  )



        with MultipartFileManager(filename, part) as manager:
            for (part_number, part_content) in enumerate(manager, part):

                self.rate_limiter.reset( queue_service.get_worker_ratelimit() )
                # FIX THIS: quantidade transferida e total sempre
                #eh reportada como o da parte

                part_file = StringIO(part_content)
                multipart.upload_part_from_file(part_file,
                                                part_number + 1,
                                                cb=self.callbacks,
                                                num_cb=-1)

                self.multipart_status_callbacks(filename, part_number)


        size = os.path.getsize(filename)
        sent_size = (part_number * MIN_MULTIPART_SIZE) + len(part_content)

        if size == sent_size:
            multipart.complete_upload()
        else:
            raise IncompleteUpload("Not sent full file size")