def test_evenly_divisible(self): self.tmp.write('01234567') chunks = utils.f_chunk(self.tmp, 2) assert len(chunks) == 4 for r in range(4): assert utils.f_sizeof(chunks[r]) == 2 assert chunks[r].read() == '{}{}'.format(r * 2, r * 2 + 1) chunks[r].close() assert chunks[r].closed
def test_evenly_divisible(self): self.tmp.write('01234567') chunks = utils.f_chunk(self.tmp, 2) assert len(chunks) == 4 for r in range(4): assert utils.f_sizeof(chunks[r]) == 2 assert chunks[r].read() == '{}{}'.format(r*2, r*2+1) chunks[r].close() assert chunks[r].closed
def test_seek_tell(self): self.tmp.write('01234567') c = utils.f_chunk(self.tmp, 4)[1] assert c.tell() == 0 c.seek(0, 2) assert c.tell() == 4 c.seek(0) assert c.tell() == 0 c.seek(2, 1) assert c.tell() == 2
def _is_unmodified(self, s3_key): local_path = self._get_local_path_from_key(s3_key.name) with open(local_path, 'rb') as f: if utils.f_sizeof(f) != s3_key.size: return False if not '-' in s3_key.md5: local_md5 = hexlify(utils.f_md5(f)) else: chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE) m = hashlib.md5() for chunk in chunks: m.update(utils.f_md5(chunk)) local_md5 = "{}-{}".format(hexlify(m.digest()), len(chunks)) return local_md5 == s3_key.md5
def _multipart_upload(self, f): upload_id = self._initiate_multipart_upload() # Upload individual parts upload_reqs = [] parts = [] chunks = utils.f_chunk(f, constants.MULTIPART_PART_SIZE) for r in range(len(chunks)): upload_reqs.append([ self._multipart_upload_part, chunks[r], r+1, upload_id ]) parts.append((r+1, hexlify(utils.f_md5(chunks[r])))) try: self.conn.join(upload_reqs) except: self._abort_multipart_upload(upload_id) raise self._complete_multipart_upload(upload_id, parts)
def _multipart_upload(self, file_like_object): self.log_upload(file_like_object) upload_id = self._initiate_multipart_upload() # Upload individual parts upload_reqs = [] parts = [] chunks = utils.f_chunk(file_like_object, constants.MULTIPART_PART_SIZE) for r in range(len(chunks)): upload_reqs.append([ self._multipart_upload_part, chunks[r], r+1, upload_id ]) parts.append((r+1, hexlify(utils.f_md5(chunks[r])))) try: self.conn.join(upload_reqs) except: self._abort_multipart_upload(upload_id) raise self._complete_multipart_upload(upload_id, parts)