def test_evenly_divisible(self):
self.tmp.write('01234567')
chunks = utils.f_chunk(self.tmp, 2)
assert len(chunks) == 4
for r in range(4):
assert utils.f_sizeof(chunks[r]) == 2
assert chunks[r].read() == '{}{}'.format(r * 2, r * 2 + 1)
chunks[r].close()
assert chunks[r].closed
def test_evenly_divisible(self):
self.tmp.write('01234567')
chunks = utils.f_chunk(self.tmp, 2)
assert len(chunks) == 4
for r in range(4):
assert utils.f_sizeof(chunks[r]) == 2
assert chunks[r].read() == '{}{}'.format(r*2, r*2+1)
chunks[r].close()
assert chunks[r].closed
def test_seek_tell(self):
self.tmp.write('01234567')
c = utils.f_chunk(self.tmp, 4)[1]
assert c.tell() == 0
c.seek(0, 2)
assert c.tell() == 4
c.seek(0)
assert c.tell() == 0
c.seek(2, 1)
assert c.tell() == 2
def test_seek_tell(self):
self.tmp.write('01234567')
c = utils.f_chunk(self.tmp, 4)[1]
assert c.tell() == 0
c.seek(0, 2)
assert c.tell() == 4
c.seek(0)
assert c.tell() == 0
c.seek(2, 1)
assert c.tell() == 2
def _is_unmodified(self, s3_key):
local_path = self._get_local_path_from_key(s3_key.name)
with open(local_path, 'rb') as f:
if utils.f_sizeof(f) != s3_key.size:
return False
if not '-' in s3_key.md5:
local_md5 = hexlify(utils.f_md5(f))
else:
chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE)
m = hashlib.md5()
for chunk in chunks:
m.update(utils.f_md5(chunk))
local_md5 = "{}-{}".format(hexlify(m.digest()), len(chunks))
return local_md5 == s3_key.md5
def _multipart_upload(self, f):
upload_id = self._initiate_multipart_upload()
# Upload individual parts
upload_reqs = []
parts = []
chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE)
for r in range(len(chunks)):
upload_reqs.append([
self._multipart_upload_part,
chunks[r],
r+1,
upload_id
])
parts.append((r+1, hexlify(utils.f_md5(chunks[r]))))
try:
self.conn.join(upload_reqs)
except:
self._abort_multipart_upload(upload_id)
raise
self._complete_multipart_upload(upload_id, parts)
def _multipart_upload(self, file_like_object):
self.log_upload(file_like_object)
upload_id = self._initiate_multipart_upload()
# Upload individual parts
upload_reqs = []
parts = []
chunks = utils.f_chunk(file_like_object, constants.MULTIPART_PART_SIZE)
for r in range(len(chunks)):
upload_reqs.append([
self._multipart_upload_part,
chunks[r],
r+1,
upload_id
])
parts.append((r+1, hexlify(utils.f_md5(chunks[r]))))
try:
self.conn.join(upload_reqs)
except:
self._abort_multipart_upload(upload_id)
raise
self._complete_multipart_upload(upload_id, parts)
