Exemplo n.º 1
0
    def test_four_part_file(self):
        key_name = "k"
        contents = "01234567890123456789"
        sfp = StringIO(contents)

        # upload 20 bytes in 4 parts of 5 bytes each
        mpu = self.bucket.initiate_multipart_upload(key_name)
        mpu.upload_part_from_file(sfp, part_num=1, size=5)
        mpu.upload_part_from_file(sfp, part_num=2, size=5)
        mpu.upload_part_from_file(sfp, part_num=3, size=5)
        mpu.upload_part_from_file(sfp, part_num=4, size=5)
        sfp.close()

        etags = {}
        pn = 0
        for part in mpu:
            pn += 1
            self.assertEqual(5, part.size)
            etags[pn] = part.etag
        self.assertEqual(pn, 4)
        # etags for 01234
        self.assertEqual(etags[1], etags[3])
        # etags for 56789
        self.assertEqual(etags[2], etags[4])
        # etag 01234 != etag 56789
        self.assertNotEqual(etags[1], etags[2])

        # parts are too small to compete as each part must
        # be a min of 5MB so so we'll assume that is enough
        # testing and abort the upload.
        mpu.cancel_upload()
Exemplo n.º 2
0
    def test_upload_part_by_size(self):
        key_name = "k"
        contents = "01234567890123456789"
        sfp = StringIO(contents)

        # upload 20 bytes in 4 parts of 5 bytes each
        mpu = self.bucket.initiate_multipart_upload(key_name)
        mpu.upload_part_from_file(sfp, part_num=1, size=5)
        mpu.upload_part_from_file(sfp, part_num=2, size=5)
        mpu.upload_part_from_file(sfp, part_num=3, size=5)
        mpu.upload_part_from_file(sfp, part_num=4, size=5)
        sfp.close()

        etags = {}
        pn = 0
        for part in mpu:
            pn += 1
            self.assertEqual(5, part.size)
            etags[pn] = part.etag
        self.assertEqual(pn, 4)
        # etags for 01234
        self.assertEqual(etags[1], etags[3])
        # etags for 56789
        self.assertEqual(etags[2], etags[4])
        # etag 01234 != etag 56789
        self.assertNotEqual(etags[1], etags[2])

        # parts are too small to complete as each part must
        # be a min of 5MB so so we'll assume that is enough
        # testing and abort the upload.
        mpu.cancel_upload()
Exemplo n.º 3
0
 def test_complete_japanese(self):
     key_name = u"テスト"
     mpu = self.bucket.initiate_multipart_upload(key_name)
     fp = StringIO("small file")
     mpu.upload_part_from_file(fp, part_num=1)
     fp.close()
     cmpu = mpu.complete_upload()
     self.assertEqual(cmpu.key_name, key_name)
     self.assertNotEqual(cmpu.etag, None)
Exemplo n.º 4
0
 def test_complete_japanese(self):
     key_name = u"テスト"
     mpu = self.bucket.initiate_multipart_upload(key_name)
     fp = StringIO("small file")
     mpu.upload_part_from_file(fp, part_num=1)
     fp.close()
     cmpu = mpu.complete_upload()
     self.assertEqual(cmpu.key_name, key_name)
     self.assertNotEqual(cmpu.etag, None)
Exemplo n.º 5
0
 def test_etag_of_parts(self):
     key_name = "etagtest"
     mpu = self.bucket.initiate_multipart_upload(key_name)
     fp = StringIO("small file")
     # upload 2 parts and save each part
     uparts = []
     uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
     uparts.append(mpu.upload_part_from_file(fp, part_num=2))
     fp.close()
     # compare uploaded parts etag to listed parts
     pn = 0
     for lpart in mpu:
         self.assertEqual(uparts[pn].etag, lpart.etag)
         pn += 1
     # Can't complete 2 small parts so just clean up.
     mpu.cancel_upload()
Exemplo n.º 6
0
 def test_etag_of_parts(self):
     key_name = "etagtest"
     mpu = self.bucket.initiate_multipart_upload(key_name)
     fp = StringIO("small file")
     # upload 2 parts and save each part
     uparts = []
     uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
     uparts.append(mpu.upload_part_from_file(fp, part_num=2))
     fp.close()
     # compare uploaded parts etag to listed parts
     pn = 0
     for lpart in mpu:
         self.assertEqual(uparts[pn].etag, lpart.etag)
         pn += 1
     # Can't complete 2 small parts so just clean up.
     mpu.cancel_upload()
Exemplo n.º 7
0
    def test_file_callback(self):
        def callback(wrote, total):
            self.my_cb_cnt += 1
            self.assertNotEqual(wrote, self.my_cb_last, "called twice with same value")
            self.my_cb_last = wrote

        # Zero bytes written => 1 call
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        sfp = StringIO("")
        k.set_contents_from_file(sfp, cb=callback, num_cb=10)
        self.assertEqual(self.my_cb_cnt, 1)
        self.assertEqual(self.my_cb_last, 0)
        sfp.close()

        # Read back zero bytes => 1 call
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback)
        self.assertEqual(self.my_cb_cnt, 1)
        self.assertEqual(self.my_cb_last, 0)

        content="01234567890123456789"
        sfp = StringIO(content)

        # expect 2 calls due start/finish
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.set_contents_from_file(sfp, cb=callback, num_cb=10)
        self.assertEqual(self.my_cb_cnt, 2)
        self.assertEqual(self.my_cb_last, 20)

        # Read back all bytes => 2 calls
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback).decode('utf-8')
        self.assertEqual(self.my_cb_cnt, 2)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # rewind sfp and try upload again. -1 should call
        # for every read/write so that should make 11 when bs=2
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=-1)
        self.assertEqual(self.my_cb_cnt, 11)
        self.assertEqual(self.my_cb_last, 20)

        # Read back all bytes => 11 calls
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=-1).decode('utf-8')
        self.assertEqual(self.my_cb_cnt, 11)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 1 times => 2 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=1)
        self.assertTrue(self.my_cb_cnt <= 2)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 1 times => 2 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=1).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 2)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 2 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=2)
        self.assertTrue(self.my_cb_cnt <= 2)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 2 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=2).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 2)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 3 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=3)
        self.assertTrue(self.my_cb_cnt <= 3)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 3 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=3).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 3)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 4 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=4)
        self.assertTrue(self.my_cb_cnt <= 4)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 4 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=4).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 4)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 6 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=6)
        self.assertTrue(self.my_cb_cnt <= 6)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 6 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=6).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 6)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 10 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=10)
        self.assertTrue(self.my_cb_cnt <= 10)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 10 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=10).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 10)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)

        # no more than 1000 times
        # last time always 20 bytes
        sfp.seek(0)
        self.my_cb_cnt = 0
        self.my_cb_last = None
        k = self.bucket.new_key("k")
        k.BufferSize = 2
        k.set_contents_from_file(sfp, cb=callback, num_cb=1000)
        self.assertTrue(self.my_cb_cnt <= 1000)
        self.assertEqual(self.my_cb_last, 20)

        # no more than 1000 times
        self.my_cb_cnt = 0
        self.my_cb_last = None
        s = k.get_contents_as_string(cb=callback, num_cb=1000).decode('utf-8')
        self.assertTrue(self.my_cb_cnt <= 1000)
        self.assertEqual(self.my_cb_last, 20)
        self.assertEqual(s, content)
Exemplo n.º 8
0
    def set_contents_from_string(self,
                                 s,
                                 headers=None,
                                 replace=True,
                                 cb=None,
                                 num_cb=10,
                                 policy=None,
                                 md5=None,
                                 if_generation=None):
        """
        Store an object in GCS using the name of the Key object as the
        key in GCS and the string 's' as the contents.
        See set_contents_from_file method for details about the
        parameters.

        :type headers: dict
        :param headers: Additional headers to pass along with the
                        request to AWS.

        :type replace: bool
        :param replace: If True, replaces the contents of the file if
                        it already exists.

        :type cb: function
        :param cb: a callback function that will be called to report
                   progress on the upload. The callback should accept
                   two integer parameters, the first representing the
                   number of bytes that have been successfully
                   transmitted to GCS and the second representing the
                   size of the to be transmitted object.

        :type cb: int
        :param num_cb: (optional) If a callback is specified with
                       the cb parameter this parameter determines the
                       granularity of the callback by defining
                       the maximum number of times the callback will
                       be called during the file transfer.

        :type policy: :class:`boto.gs.acl.CannedACLStrings`
        :param policy: A canned ACL policy that will be applied to the
                       new key in GCS.

        :type md5: A tuple containing the hexdigest version of the MD5
                   checksum of the file as the first element and the
                   Base64-encoded version of the plain checksum as the
                   second element. This is the same format returned by
                   the compute_md5 method.
        :param md5: If you need to compute the MD5 for any reason prior
                    to upload, it's silly to have to do it twice so this
                    param, if present, will be used as the MD5 values
                    of the file. Otherwise, the checksum will be computed.

        :type if_generation: int
        :param if_generation: (optional) If set to a generation number, the
            object will only be written to if its current generation number is
            this value. If set to the value 0, the object will only be written
            if it doesn't already exist.
        """

        # Clear out any previously computed md5 hashes, since we are setting the content.
        self.md5 = None
        self.base64md5 = None

        fp = StringIO(get_utf8_value(s))
        r = self.set_contents_from_file(fp,
                                        headers,
                                        replace,
                                        cb,
                                        num_cb,
                                        policy,
                                        md5,
                                        if_generation=if_generation)
        fp.close()
        return r
Exemplo n.º 9
0
Arquivo: key.py Projeto: C2Devel/boto
    def set_contents_from_string(self, s, headers=None, replace=True,
                                 cb=None, num_cb=10, policy=None, md5=None,
                                 if_generation=None):
        """
        Store an object in GCS using the name of the Key object as the
        key in GCS and the string 's' as the contents.
        See set_contents_from_file method for details about the
        parameters.

        :type headers: dict
        :param headers: Additional headers to pass along with the
                        request to AWS.

        :type replace: bool
        :param replace: If True, replaces the contents of the file if
                        it already exists.

        :type cb: function
        :param cb: a callback function that will be called to report
                   progress on the upload. The callback should accept
                   two integer parameters, the first representing the
                   number of bytes that have been successfully
                   transmitted to GCS and the second representing the
                   size of the to be transmitted object.

        :type cb: int
        :param num_cb: (optional) If a callback is specified with
                       the cb parameter this parameter determines the
                       granularity of the callback by defining
                       the maximum number of times the callback will
                       be called during the file transfer.

        :type policy: :class:`boto.gs.acl.CannedACLStrings`
        :param policy: A canned ACL policy that will be applied to the
                       new key in GCS.

        :type md5: A tuple containing the hexdigest version of the MD5
                   checksum of the file as the first element and the
                   Base64-encoded version of the plain checksum as the
                   second element. This is the same format returned by
                   the compute_md5 method.
        :param md5: If you need to compute the MD5 for any reason prior
                    to upload, it's silly to have to do it twice so this
                    param, if present, will be used as the MD5 values
                    of the file. Otherwise, the checksum will be computed.

        :type if_generation: int
        :param if_generation: (optional) If set to a generation number, the
            object will only be written to if its current generation number is
            this value. If set to the value 0, the object will only be written
            if it doesn't already exist.
        """

        # Clear out any previously computed md5 hashes, since we are setting the content.
        self.md5 = None
        self.base64md5 = None

        fp = StringIO(get_utf8_value(s))
        r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
                                        policy, md5,
                                        if_generation=if_generation)
        fp.close()
        return r