Exemple #1
0
 def _calculate_required_part_size(self, total_size):
     min_part_size_required = minimum_part_size(total_size)
     if self._part_size >= min_part_size_required:
         part_size = self._part_size
     else:
         part_size = min_part_size_required
         log.debug("The part size specified (%s) is smaller than "
                   "the minimum required part size.  Using a part "
                   "size of: %s", self._part_size, part_size)
     total_parts = int(math.ceil(total_size / float(part_size)))
     return total_parts, part_size
Exemple #2
0
 def _calculate_required_part_size(self, total_size):
     min_part_size_required = minimum_part_size(total_size)
     if self._part_size >= min_part_size_required:
         part_size = self._part_size
     else:
         part_size = min_part_size_required
         log.debug("The part size specified (%s) is smaller than "
                   "the minimum required part size.  Using a part "
                   "size of: %s", self._part_size, part_size)
     total_parts = int(math.ceil(total_size / float(part_size)))
     return total_parts, part_size
Exemple #3
0
    def create_archive_from_file(self,
                                 filename=None,
                                 file_obj=None,
                                 description=None,
                                 upload_id_callback=None):
        """
        Create a new archive and upload the data from the given file
        or file-like object.

        :type filename: str
        :param filename: A filename to upload

        :type file_obj: file
        :param file_obj: A file-like object to upload

        :type description: str
        :param description: An optional description for the archive.

        :type upload_id_callback: function
        :param upload_id_callback: if set, call with the upload_id as the
            only parameter when it becomes known, to enable future calls
            to resume_archive_from_file in case resume is needed.

        :rtype: str
        :return: The archive id of the newly created archive
        """
        part_size = self.DefaultPartSize
        if not file_obj:
            file_size = os.path.getsize(filename)
            try:
                part_size = minimum_part_size(file_size, part_size)
            except ValueError:
                raise UploadArchiveError("File size of %s bytes exceeds "
                                         "40,000 GB archive limit of Glacier.")
            file_obj = open(filename, "rb")
        writer = self.create_archive_writer(description=description,
                                            part_size=part_size)
        if upload_id_callback:
            upload_id_callback(writer.upload_id)
        while True:
            data = file_obj.read(part_size)
            if not data:
                break
            writer.write(data)
        writer.close()
        return writer.get_archive_id()
Exemple #4
0
    def create_archive_from_file(self, filename=None, file_obj=None,
                                 description=None, upload_id_callback=None):
        """
        Create a new archive and upload the data from the given file
        or file-like object.

        :type filename: str
        :param filename: A filename to upload

        :type file_obj: file
        :param file_obj: A file-like object to upload

        :type description: str
        :param description: An optional description for the archive.

        :type upload_id_callback: function
        :param upload_id_callback: if set, call with the upload_id as the
            only parameter when it becomes known, to enable future calls
            to resume_archive_from_file in case resume is needed.

        :rtype: str
        :return: The archive id of the newly created archive
        """
        part_size = self.DefaultPartSize
        if not file_obj:
            file_size = os.path.getsize(filename)
            try:
                part_size = minimum_part_size(file_size, part_size)
            except ValueError:
                raise UploadArchiveError("File size of %s bytes exceeds "
                                         "40,000 GB archive limit of Glacier.")
            file_obj = open(filename, "rb")
        writer = self.create_archive_writer(
            description=description,
            part_size=part_size)
        if upload_id_callback:
            upload_id_callback(writer.upload_id)
        while True:
            data = file_obj.read(part_size)
            if not data:
                break
            writer.write(data)
        writer.close()
        return writer.get_archive_id()
Exemple #5
0
 def test_default_part_size_can_be_specified(self):
     default_part_size = 2 * 1024 * 1024
     self.assertEqual(minimum_part_size(8 * 1024 * 1024, default_part_size),
                      default_part_size)
Exemple #6
0
 def test_file_size_too_large(self):
     with self.assertRaises(ValueError):
         minimum_part_size((40000 * 1024 * 1024 * 1024) + 1)
Exemple #7
0
 def test_terabyte_size(self):
     # For a 4 TB file we need at least a 512 MB part size.
     self.assertEqual(minimum_part_size(4 * 1024 * 1024 * 1024 * 1024),
                      512 * 1024 * 1024)
Exemple #8
0
 def test_gigabyte_size(self):
     # If we're over the maximum default part size, we go up to the next
     # power of two until we find a part size that keeps us under 10,000
     # parts.
     self.assertEqual(minimum_part_size(8 * 1024 * 1024 * 10000),
                      8 * 1024 * 1024)
Exemple #9
0
 def test_under_the_maximum_value(self):
     # If we're under the maximum, we can use 4MB part sizes.
     self.assertEqual(minimum_part_size(8 * 1024 * 1024), 4 * 1024 * 1024)
Exemple #10
0
 def test_small_values_still_use_default_part_size(self):
     self.assertEqual(minimum_part_size(1), 4 * 1024 * 1024)
Exemple #11
0
 def test_default_part_size_can_be_specified(self):
     default_part_size = 2 * 1024 * 1024
     self.assertEqual(minimum_part_size(8 * 1024 * 1024, default_part_size),
                      default_part_size)
Exemple #12
0
 def test_file_size_too_large(self):
     with self.assertRaises(ValueError):
         minimum_part_size((40000 * 1024 * 1024 * 1024) + 1)
Exemple #13
0
 def test_terabyte_size(self):
     # For a 4 TB file we need at least a 512 MB part size.
     self.assertEqual(minimum_part_size(4 * 1024 * 1024 * 1024 * 1024),
                      512 * 1024 * 1024)
Exemple #14
0
 def test_gigabyte_size(self):
     # If we're over the maximum default part size, we go up to the next
     # power of two until we find a part size that keeps us under 10,000
     # parts.
     self.assertEqual(minimum_part_size(8 * 1024 * 1024 * 10000),
                      8 * 1024 * 1024)
Exemple #15
0
 def test_under_the_maximum_value(self):
     # If we're under the maximum, we can use 4MB part sizes.
     self.assertEqual(minimum_part_size(8 * 1024 * 1024),
                      4 * 1024 * 1024)
Exemple #16
0
 def test_small_values_still_use_default_part_size(self):
     self.assertEqual(minimum_part_size(1), 4 * 1024 * 1024)