def test_four_part_file(self):
key_name = "k"
contents = "01234567890123456789"
sfp = StringIO(contents)
# upload 20 bytes in 4 parts of 5 bytes each
mpu = self.bucket.initiate_multipart_upload(key_name)
mpu.upload_part_from_file(sfp, part_num=1, size=5)
mpu.upload_part_from_file(sfp, part_num=2, size=5)
mpu.upload_part_from_file(sfp, part_num=3, size=5)
mpu.upload_part_from_file(sfp, part_num=4, size=5)
sfp.close()
etags = {}
pn = 0
for part in mpu:
pn += 1
self.assertEqual(5, part.size)
etags[pn] = part.etag
self.assertEqual(pn, 4)
# etags for 01234
self.assertEqual(etags[1], etags[3])
# etags for 56789
self.assertEqual(etags[2], etags[4])
# etag 01234 != etag 56789
self.assertNotEqual(etags[1], etags[2])
# parts are too small to compete as each part must
# be a min of 5MB so so we'll assume that is enough
# testing and abort the upload.
mpu.cancel_upload()
def test_upload_part_by_size(self):
key_name = "k"
contents = "01234567890123456789"
sfp = StringIO(contents)
# upload 20 bytes in 4 parts of 5 bytes each
mpu = self.bucket.initiate_multipart_upload(key_name)
mpu.upload_part_from_file(sfp, part_num=1, size=5)
mpu.upload_part_from_file(sfp, part_num=2, size=5)
mpu.upload_part_from_file(sfp, part_num=3, size=5)
mpu.upload_part_from_file(sfp, part_num=4, size=5)
sfp.close()
etags = {}
pn = 0
for part in mpu:
pn += 1
self.assertEqual(5, part.size)
etags[pn] = part.etag
self.assertEqual(pn, 4)
# etags for 01234
self.assertEqual(etags[1], etags[3])
# etags for 56789
self.assertEqual(etags[2], etags[4])
# etag 01234 != etag 56789
self.assertNotEqual(etags[1], etags[2])
# parts are too small to complete as each part must
# be a min of 5MB so so we'll assume that is enough
# testing and abort the upload.
mpu.cancel_upload()
def test_complete_japanese(self):
key_name = u"テスト"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
mpu.upload_part_from_file(fp, part_num=1)
fp.close()
cmpu = mpu.complete_upload()
self.assertEqual(cmpu.key_name, key_name)
self.assertNotEqual(cmpu.etag, None)
def test_complete_japanese(self):
key_name = u"テスト"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
mpu.upload_part_from_file(fp, part_num=1)
fp.close()
cmpu = mpu.complete_upload()
self.assertEqual(cmpu.key_name, key_name)
self.assertNotEqual(cmpu.etag, None)
def test_etag_of_parts(self):
key_name = "etagtest"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
# upload 2 parts and save each part
uparts = []
uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
uparts.append(mpu.upload_part_from_file(fp, part_num=2))
fp.close()
# compare uploaded parts etag to listed parts
pn = 0
for lpart in mpu:
self.assertEqual(uparts[pn].etag, lpart.etag)
pn += 1
# Can't complete 2 small parts so just clean up.
mpu.cancel_upload()
def test_etag_of_parts(self):
key_name = "etagtest"
mpu = self.bucket.initiate_multipart_upload(key_name)
fp = StringIO("small file")
# upload 2 parts and save each part
uparts = []
uparts.append(mpu.upload_part_from_file(fp, part_num=1, size=5))
uparts.append(mpu.upload_part_from_file(fp, part_num=2))
fp.close()
# compare uploaded parts etag to listed parts
pn = 0
for lpart in mpu:
self.assertEqual(uparts[pn].etag, lpart.etag)
pn += 1
# Can't complete 2 small parts so just clean up.
mpu.cancel_upload()
def test_file_callback(self):
def callback(wrote, total):
self.my_cb_cnt += 1
self.assertNotEqual(wrote, self.my_cb_last, "called twice with same value")
self.my_cb_last = wrote
# Zero bytes written => 1 call
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
sfp = StringIO("")
k.set_contents_from_file(sfp, cb=callback, num_cb=10)
self.assertEqual(self.my_cb_cnt, 1)
self.assertEqual(self.my_cb_last, 0)
sfp.close()
# Read back zero bytes => 1 call
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback)
self.assertEqual(self.my_cb_cnt, 1)
self.assertEqual(self.my_cb_last, 0)
content="01234567890123456789"
sfp = StringIO(content)
# expect 2 calls due start/finish
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.set_contents_from_file(sfp, cb=callback, num_cb=10)
self.assertEqual(self.my_cb_cnt, 2)
self.assertEqual(self.my_cb_last, 20)
# Read back all bytes => 2 calls
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback).decode('utf-8')
self.assertEqual(self.my_cb_cnt, 2)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# rewind sfp and try upload again. -1 should call
# for every read/write so that should make 11 when bs=2
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=-1)
self.assertEqual(self.my_cb_cnt, 11)
self.assertEqual(self.my_cb_last, 20)
# Read back all bytes => 11 calls
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=-1).decode('utf-8')
self.assertEqual(self.my_cb_cnt, 11)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 1 times => 2 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=1)
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
# no more than 1 times => 2 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=1).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 2 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=2)
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
# no more than 2 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=2).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 2)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 3 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=3)
self.assertTrue(self.my_cb_cnt <= 3)
self.assertEqual(self.my_cb_last, 20)
# no more than 3 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=3).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 3)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 4 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=4)
self.assertTrue(self.my_cb_cnt <= 4)
self.assertEqual(self.my_cb_last, 20)
# no more than 4 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=4).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 4)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 6 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=6)
self.assertTrue(self.my_cb_cnt <= 6)
self.assertEqual(self.my_cb_last, 20)
# no more than 6 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=6).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 6)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 10 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=10)
self.assertTrue(self.my_cb_cnt <= 10)
self.assertEqual(self.my_cb_last, 20)
# no more than 10 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=10).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 10)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
# no more than 1000 times
# last time always 20 bytes
sfp.seek(0)
self.my_cb_cnt = 0
self.my_cb_last = None
k = self.bucket.new_key("k")
k.BufferSize = 2
k.set_contents_from_file(sfp, cb=callback, num_cb=1000)
self.assertTrue(self.my_cb_cnt <= 1000)
self.assertEqual(self.my_cb_last, 20)
# no more than 1000 times
self.my_cb_cnt = 0
self.my_cb_last = None
s = k.get_contents_as_string(cb=callback, num_cb=1000).decode('utf-8')
self.assertTrue(self.my_cb_cnt <= 1000)
self.assertEqual(self.my_cb_last, 20)
self.assertEqual(s, content)
def set_contents_from_string(self,
s,
headers=None,
replace=True,
cb=None,
num_cb=10,
policy=None,
md5=None,
if_generation=None):
"""
Store an object in GCS using the name of the Key object as the
key in GCS and the string 's' as the contents.
See set_contents_from_file method for details about the
parameters.
:type headers: dict
:param headers: Additional headers to pass along with the
request to AWS.
:type replace: bool
:param replace: If True, replaces the contents of the file if
it already exists.
:type cb: function
:param cb: a callback function that will be called to report
progress on the upload. The callback should accept
two integer parameters, the first representing the
number of bytes that have been successfully
transmitted to GCS and the second representing the
size of the to be transmitted object.
:type cb: int
:param num_cb: (optional) If a callback is specified with
the cb parameter this parameter determines the
granularity of the callback by defining
the maximum number of times the callback will
be called during the file transfer.
:type policy: :class:`boto.gs.acl.CannedACLStrings`
:param policy: A canned ACL policy that will be applied to the
new key in GCS.
:type md5: A tuple containing the hexdigest version of the MD5
checksum of the file as the first element and the
Base64-encoded version of the plain checksum as the
second element. This is the same format returned by
the compute_md5 method.
:param md5: If you need to compute the MD5 for any reason prior
to upload, it's silly to have to do it twice so this
param, if present, will be used as the MD5 values
of the file. Otherwise, the checksum will be computed.
:type if_generation: int
:param if_generation: (optional) If set to a generation number, the
object will only be written to if its current generation number is
this value. If set to the value 0, the object will only be written
if it doesn't already exist.
"""
# Clear out any previously computed md5 hashes, since we are setting the content.
self.md5 = None
self.base64md5 = None
fp = StringIO(get_utf8_value(s))
r = self.set_contents_from_file(fp,
headers,
replace,
cb,
num_cb,
policy,
md5,
if_generation=if_generation)
fp.close()
return r
def set_contents_from_string(self, s, headers=None, replace=True,
cb=None, num_cb=10, policy=None, md5=None,
if_generation=None):
"""
Store an object in GCS using the name of the Key object as the
key in GCS and the string 's' as the contents.
See set_contents_from_file method for details about the
parameters.
:type headers: dict
:param headers: Additional headers to pass along with the
request to AWS.
:type replace: bool
:param replace: If True, replaces the contents of the file if
it already exists.
:type cb: function
:param cb: a callback function that will be called to report
progress on the upload. The callback should accept
two integer parameters, the first representing the
number of bytes that have been successfully
transmitted to GCS and the second representing the
size of the to be transmitted object.
:type cb: int
:param num_cb: (optional) If a callback is specified with
the cb parameter this parameter determines the
granularity of the callback by defining
the maximum number of times the callback will
be called during the file transfer.
:type policy: :class:`boto.gs.acl.CannedACLStrings`
:param policy: A canned ACL policy that will be applied to the
new key in GCS.
:type md5: A tuple containing the hexdigest version of the MD5
checksum of the file as the first element and the
Base64-encoded version of the plain checksum as the
second element. This is the same format returned by
the compute_md5 method.
:param md5: If you need to compute the MD5 for any reason prior
to upload, it's silly to have to do it twice so this
param, if present, will be used as the MD5 values
of the file. Otherwise, the checksum will be computed.
:type if_generation: int
:param if_generation: (optional) If set to a generation number, the
object will only be written to if its current generation number is
this value. If set to the value 0, the object will only be written
if it doesn't already exist.
"""
# Clear out any previously computed md5 hashes, since we are setting the content.
self.md5 = None
self.base64md5 = None
fp = StringIO(get_utf8_value(s))
r = self.set_contents_from_file(fp, headers, replace, cb, num_cb,
policy, md5,
if_generation=if_generation)
fp.close()
return r