Example #1
0
 def test_chunks_with_leftovers(self):
     bytestring = 'a' * (2 * 1024 * 1024 + 20)
     chunks = chunk_hashes(bytestring)
     self.assertEqual(len(chunks), 3)
     self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())
     self.assertEqual(chunks[1], sha256('a' * 1024 * 1024).digest())
     self.assertEqual(chunks[2], sha256('a' * 20).digest())
Example #2
0
 def test_chunks_with_leftovers(self):
     bytestring = 'a' * (2 * 1024 * 1024 + 20)
     chunks = chunk_hashes(bytestring)
     self.assertEqual(len(chunks), 3)
     self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())
     self.assertEqual(chunks[1], sha256('a' * 1024 * 1024).digest())
     self.assertEqual(chunks[2], sha256('a' * 20).digest())
Example #3
0
    def multipart_put(self, upload_id, part_num, part):
        fileobj = open(part, 'rb')
        bytes_to_upload = fileobj.read(self._part_size)
        part_size = os.fstat(fileobj.fileno()).st_size

        start_byte = part_num * self._part_size
        content_range = (start_byte, start_byte + part_size - 1)
        linear_hash = hashlib.sha256(bytes_to_upload).hexdigest()
        part_tree_hash = tree_hash(chunk_hashes(bytes_to_upload))
        hex_part_tree_hash = bytes_to_hex(part_tree_hash)

        self._conn.upload_part(
                self._vault_name,
                upload_id,
                linear_hash,
                hex_part_tree_hash,
                content_range,
                bytes_to_upload
        )

        if part_num not in self._part_num_list:
            self._part_num_list.append(part_num)
            self._tree_hashes.append(part_tree_hash)
            self._total_size += part_size

        fileobj.close()
Example #4
0
    def multipart_put(self, upload_id, part_num, part):
        fileobj = open(part, 'rb')
        bytes_to_upload = fileobj.read(self._part_size)
        part_size = os.fstat(fileobj.fileno()).st_size

        start_byte = part_num * self._part_size
        content_range = (start_byte, start_byte + part_size - 1)
        linear_hash = hashlib.sha256(bytes_to_upload).hexdigest()
        part_tree_hash = tree_hash(chunk_hashes(bytes_to_upload))
        hex_part_tree_hash = bytes_to_hex(part_tree_hash)

        self._conn.upload_part(
                self._vault_name,
                upload_id,
                linear_hash,
                hex_part_tree_hash,
                content_range,
                bytes_to_upload
        )

        if part_num not in self._part_num_list:
            self._part_num_list.append(part_num)
            self._tree_hashes.append(part_tree_hash)
            self._total_size += part_size

        fileobj.close()
Example #5
0
def calculate_mock_vault_calls(data, part_size, chunk_size):
    upload_part_calls = []
    data_tree_hashes = []
    for i, data_part in enumerate(partify(data, part_size)):
        start = i * part_size
        end = start + len(data_part)
        data_part_tree_hash_blob = tree_hash(
            chunk_hashes(data_part, chunk_size))
        data_part_tree_hash = bytes_to_hex(data_part_tree_hash_blob)
        data_part_linear_hash = sha256(data_part).hexdigest()
        upload_part_calls.append(
            call.layer1.upload_part(sentinel.vault_name, sentinel.upload_id,
                                    data_part_linear_hash, data_part_tree_hash,
                                    (start, end - 1), data_part))
        data_tree_hashes.append(data_part_tree_hash_blob)

    return upload_part_calls, data_tree_hashes
Example #6
0
def calculate_mock_vault_calls(data, part_size, chunk_size):
    upload_part_calls = []
    data_tree_hashes = []
    for i, data_part in enumerate(partify(data, part_size)):
        start = i * part_size
        end = start + len(data_part)
        data_part_tree_hash_blob = tree_hash(
            chunk_hashes(data_part, chunk_size))
        data_part_tree_hash = bytes_to_hex(data_part_tree_hash_blob)
        data_part_linear_hash = sha256(data_part).hexdigest()
        upload_part_calls.append(
            call.layer1.upload_part(
                sentinel.vault_name, sentinel.upload_id,
                data_part_linear_hash, data_part_tree_hash,
                (start, end - 1), data_part))
        data_tree_hashes.append(data_part_tree_hash_blob)

    return upload_part_calls, data_tree_hashes
Example #7
0
    def check_no_resume(self, data, resume_set=set()):
        fobj = StringIO(data)
        part_hash_map = {}
        for part_index in resume_set:
            start = self.part_size * part_index
            end = start + self.part_size
            part_data = data[start:end]
            part_hash_map[part_index] = tree_hash(
                chunk_hashes(part_data, self.chunk_size))

        resume_file_upload(
            self.vault, sentinel.upload_id, self.part_size, fobj,
            part_hash_map, self.chunk_size)

        upload_part_calls, data_tree_hashes = calculate_mock_vault_calls(
            data, self.part_size, self.chunk_size)
        resume_upload_part_calls = [
            call for part_index, call in enumerate(upload_part_calls)
                    if part_index not in resume_set]
        check_mock_vault_calls(
            self.vault, resume_upload_part_calls, data_tree_hashes, len(data))
Example #8
0
    def check_no_resume(self, data, resume_set=set()):
        fobj = StringIO(data)
        part_hash_map = {}
        for part_index in resume_set:
            start = self.part_size * part_index
            end = start + self.part_size
            part_data = data[start:end]
            part_hash_map[part_index] = tree_hash(
                chunk_hashes(part_data, self.chunk_size))

        resume_file_upload(self.vault, sentinel.upload_id, self.part_size,
                           fobj, part_hash_map, self.chunk_size)

        upload_part_calls, data_tree_hashes = calculate_mock_vault_calls(
            data, self.part_size, self.chunk_size)
        resume_upload_part_calls = [
            call for part_index, call in enumerate(upload_part_calls)
            if part_index not in resume_set
        ]
        check_mock_vault_calls(self.vault, resume_upload_part_calls,
                               data_tree_hashes, len(data))
Example #9
0
 def test_less_than_one_chunk(self):
     chunks = chunk_hashes('aaaa')
     self.assertEqual(len(chunks), 1)
     self.assertEqual(chunks[0], sha256('aaaa').digest())
Example #10
0
 def test_chunk_hashes_exact(self):
     chunks = chunk_hashes('a' * (2 * 1024 * 1024))
     self.assertEqual(len(chunks), 2)
     self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())
Example #11
0
 def test_chunk_hashes_exact(self):
     chunks = chunk_hashes('a' * (2 * 1024 * 1024))
     self.assertEqual(len(chunks), 2)
     self.assertEqual(chunks[0], sha256('a' * 1024 * 1024).digest())
Example #12
0
 def test_less_than_one_chunk(self):
     chunks = chunk_hashes('aaaa')
     self.assertEqual(len(chunks), 1)
     self.assertEqual(chunks[0], sha256('aaaa').digest())