def test_chunks_with_leftovers(self): bytestring = 'a' * (2 * 1024 * 1024 + 20) chunks = chunk_hashes(bytestring) self.assertEqual(len(chunks), 3) self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest()) self.assertEqual(chunks[1], sha256('a' * 1024 * 1024).digest()) self.assertEqual(chunks[2], sha256('a' * 20).digest())
def multipart_put(self, upload_id, part_num, part): fileobj = open(part, 'rb') bytes_to_upload = fileobj.read(self._part_size) part_size = os.fstat(fileobj.fileno()).st_size start_byte = part_num * self._part_size content_range = (start_byte, start_byte + part_size - 1) linear_hash = hashlib.sha256(bytes_to_upload).hexdigest() part_tree_hash = tree_hash(chunk_hashes(bytes_to_upload)) hex_part_tree_hash = bytes_to_hex(part_tree_hash) self._conn.upload_part( self._vault_name, upload_id, linear_hash, hex_part_tree_hash, content_range, bytes_to_upload ) if part_num not in self._part_num_list: self._part_num_list.append(part_num) self._tree_hashes.append(part_tree_hash) self._total_size += part_size fileobj.close()
def calculate_mock_vault_calls(data, part_size, chunk_size): upload_part_calls = [] data_tree_hashes = [] for i, data_part in enumerate(partify(data, part_size)): start = i * part_size end = start + len(data_part) data_part_tree_hash_blob = tree_hash( chunk_hashes(data_part, chunk_size)) data_part_tree_hash = bytes_to_hex(data_part_tree_hash_blob) data_part_linear_hash = sha256(data_part).hexdigest() upload_part_calls.append( call.layer1.upload_part(sentinel.vault_name, sentinel.upload_id, data_part_linear_hash, data_part_tree_hash, (start, end - 1), data_part)) data_tree_hashes.append(data_part_tree_hash_blob) return upload_part_calls, data_tree_hashes
def calculate_mock_vault_calls(data, part_size, chunk_size): upload_part_calls = [] data_tree_hashes = [] for i, data_part in enumerate(partify(data, part_size)): start = i * part_size end = start + len(data_part) data_part_tree_hash_blob = tree_hash( chunk_hashes(data_part, chunk_size)) data_part_tree_hash = bytes_to_hex(data_part_tree_hash_blob) data_part_linear_hash = sha256(data_part).hexdigest() upload_part_calls.append( call.layer1.upload_part( sentinel.vault_name, sentinel.upload_id, data_part_linear_hash, data_part_tree_hash, (start, end - 1), data_part)) data_tree_hashes.append(data_part_tree_hash_blob) return upload_part_calls, data_tree_hashes
def check_no_resume(self, data, resume_set=set()): fobj = StringIO(data) part_hash_map = {} for part_index in resume_set: start = self.part_size * part_index end = start + self.part_size part_data = data[start:end] part_hash_map[part_index] = tree_hash( chunk_hashes(part_data, self.chunk_size)) resume_file_upload( self.vault, sentinel.upload_id, self.part_size, fobj, part_hash_map, self.chunk_size) upload_part_calls, data_tree_hashes = calculate_mock_vault_calls( data, self.part_size, self.chunk_size) resume_upload_part_calls = [ call for part_index, call in enumerate(upload_part_calls) if part_index not in resume_set] check_mock_vault_calls( self.vault, resume_upload_part_calls, data_tree_hashes, len(data))
def check_no_resume(self, data, resume_set=set()): fobj = StringIO(data) part_hash_map = {} for part_index in resume_set: start = self.part_size * part_index end = start + self.part_size part_data = data[start:end] part_hash_map[part_index] = tree_hash( chunk_hashes(part_data, self.chunk_size)) resume_file_upload(self.vault, sentinel.upload_id, self.part_size, fobj, part_hash_map, self.chunk_size) upload_part_calls, data_tree_hashes = calculate_mock_vault_calls( data, self.part_size, self.chunk_size) resume_upload_part_calls = [ call for part_index, call in enumerate(upload_part_calls) if part_index not in resume_set ] check_mock_vault_calls(self.vault, resume_upload_part_calls, data_tree_hashes, len(data))
def test_less_than_one_chunk(self): chunks = chunk_hashes('aaaa') self.assertEqual(len(chunks), 1) self.assertEqual(chunks[0], sha256('aaaa').digest())
def test_chunk_hashes_exact(self): chunks = chunk_hashes('a' * (2 * 1024 * 1024)) self.assertEqual(len(chunks), 2) self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())