def _destroy(self, collection_id, key, unified_id_to_delete, timestamp, segment_num): user_request_id = uuid.uuid1().hex archive_priority = create_priority() unified_id_factory = UnifiedIDFactory(1) unified_id = unified_id_factory.next() message = { "message-type": "destroy-key", "priority": archive_priority, "user-request-id": user_request_id, "collection-id": collection_id, "key": key, "unified-id-to-delete": unified_id_to_delete, "unified-id": unified_id, "timestamp-repr": repr(timestamp), "segment-num": segment_num, "source-node-name": _local_node_name, "handoff-node-name": None, } reply = send_request_and_get_reply(_local_node_name, _data_writer_address, _local_node_name, _client_address, message) self.assertEqual(reply["message-type"], "destroy-key-reply") self.assertEqual(reply["user-request-id"], user_request_id) return reply
def _destroy(self, collection_id, key, unified_id_to_delete, timestamp, segment_num): user_request_id = uuid.uuid1().hex archive_priority = create_priority() unified_id_factory = UnifiedIDFactory(1) unified_id = unified_id_factory.next() message = { "message-type" : "destroy-key", "priority" : archive_priority, "user-request-id" : user_request_id, "collection-id" : collection_id, "key" : key, "unified-id-to-delete" : unified_id_to_delete, "unified-id" : unified_id, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, "source-node-name" : _local_node_name, "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message ) self.assertEqual(reply["message-type"], "destroy-key-reply") self.assertEqual(reply["user-request-id"], user_request_id) return reply
def xxxtest_archive_key_entire_with_meta(self): """ test archiving a key in a single message, including meta data """ file_size = 10 * 64 * 1024 content_item = random_string(file_size) user_request_id = uuid.uuid1().hex collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() timestamp = create_timestamp() segment_num = 2 meta_key = "".join([nimbus_meta_prefix, "test_key"]) meta_value = "pork" file_adler32 = zlib.adler32(content_item) file_md5 = hashlib.md5(content_item) unified_id_factory = UnifiedIDFactory(1) unified_id = unified_id_factory.next() message = { "message-type" : "archive-key-entire", "priority" : archive_priority, "user-request-id" : user_request_id, "collection-id" : collection_id, "key" : key, "unified-id" : unified_id, "timestamp-repr" : repr(timestamp), "conjoined-part" : 0, "segment-num" : segment_num, "segment-size" : file_size, "zfec-padding-size" : 4, "segment-adler32" : file_adler32, "segment-md5-digest": b64encode(file_md5.digest()), "file-size" : file_size, "file-adler32" : file_adler32, "file-hash" : b64encode(file_md5.digest()), "source-node-name" : _local_node_name, "handoff-node-name" : None, meta_key : meta_value } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=content_item ) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"])
def xxxtest_destroy_tombstone(self): """test destroying a key that has already been destroyed""" file_size = 10 * 64 * 1024 content_item = random_string(file_size) message_id = uuid.uuid1().hex collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() archive_timestamp = create_timestamp() destroy_1_timestamp = archive_timestamp + timedelta(seconds=1) destroy_2_timestamp = destroy_1_timestamp + timedelta(seconds=1) segment_num = 2 file_adler32 = zlib.adler32(content_item) file_md5 = hashlib.md5(content_item) message = { "message-type" : "archive-key-entire", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "timestamp-repr" : repr(archive_timestamp), "segment-num" : segment_num, "segment-size" : file_size, "segment-adler32" : file_adler32, "segment-md5-digest": b64encode(file_md5.digest()), "file-size" : file_size, "file-adler32" : file_adler32, "file-hash" : b64encode(file_md5.digest()), "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=content_item ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success") reply = self._destroy( collection_id, key, destroy_1_timestamp, segment_num ) self.assertEqual(reply["result"], "success", reply["error-message"]) reply = self._destroy( collection_id, key, destroy_2_timestamp, segment_num ) self.assertEqual(reply["result"], "success", reply["error-message"])
def xxxtest_archive_key_entire_with_meta(self): """ test archiving a key in a single message, including meta data """ file_size = 10 * 64 * 1024 content_item = random_string(file_size) user_request_id = uuid.uuid1().hex collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() timestamp = create_timestamp() segment_num = 2 meta_key = "".join([nimbus_meta_prefix, "test_key"]) meta_value = "pork" file_adler32 = zlib.adler32(content_item) file_md5 = hashlib.md5(content_item) unified_id_factory = UnifiedIDFactory(1) unified_id = unified_id_factory.next() message = { "message-type": "archive-key-entire", "priority": archive_priority, "user-request-id": user_request_id, "collection-id": collection_id, "key": key, "unified-id": unified_id, "timestamp-repr": repr(timestamp), "conjoined-part": 0, "segment-num": segment_num, "segment-size": file_size, "zfec-padding-size": 4, "segment-adler32": file_adler32, "segment-md5-digest": b64encode(file_md5.digest()), "file-size": file_size, "file-adler32": file_adler32, "file-hash": b64encode(file_md5.digest()), "source-node-name": _local_node_name, "handoff-node-name": None, meta_key: meta_value } reply = send_request_and_get_reply(_local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=content_item) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"])
def xxxtest_destroy_tombstone(self): """test destroying a key that has already been destroyed""" file_size = 10 * 64 * 1024 content_item = random_string(file_size) message_id = uuid.uuid1().hex collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() archive_timestamp = create_timestamp() destroy_1_timestamp = archive_timestamp + timedelta(seconds=1) destroy_2_timestamp = destroy_1_timestamp + timedelta(seconds=1) segment_num = 2 file_adler32 = zlib.adler32(content_item) file_md5 = hashlib.md5(content_item) message = { "message-type": "archive-key-entire", "message-id": message_id, "priority": archive_priority, "collection-id": collection_id, "key": key, "timestamp-repr": repr(archive_timestamp), "segment-num": segment_num, "segment-size": file_size, "segment-adler32": file_adler32, "segment-md5-digest": b64encode(file_md5.digest()), "file-size": file_size, "file-adler32": file_adler32, "file-hash": b64encode(file_md5.digest()), "handoff-node-name": None, } reply = send_request_and_get_reply(_local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=content_item) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success") reply = self._destroy(collection_id, key, destroy_1_timestamp, segment_num) self.assertEqual(reply["result"], "success", reply["error-message"]) reply = self._destroy(collection_id, key, destroy_2_timestamp, segment_num) self.assertEqual(reply["result"], "success", reply["error-message"])
def test_archive_key_entire(self): """test archiving all data for a key in a single message""" file_size = 10 * 64 * 1024 content_item = random_string(file_size) message_id = uuid.uuid1().hex collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() timestamp = create_timestamp() segment_num = 2 file_adler32 = zlib.adler32(content_item) file_md5 = hashlib.md5(content_item) message = { "message-type" : "archive-key-entire", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, "segment-size" : file_size, "segment-adler32" : file_adler32, "segment-md5-digest": b64encode(file_md5.digest()), "file-size" : file_size, "file-adler32" : file_adler32, "file-hash" : b64encode(file_md5.digest()), "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=content_item ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success")
def _purge(self, collection_id, key, timestamp, segment_num): message_id = uuid.uuid1().hex archive_priority = create_priority() message = { "message-type" : "purge-key", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "purge-key-reply") return reply
def xxxtest_large_archive(self): """ test archiving a file that needs more than one message. For example, a 10 Mb file: each node would get 10 120kb zefec shares. """ slice_size = 1024 * 1024 slice_count = 10 total_size = slice_size * slice_count test_data = random_string(total_size) user_request_id = uuid.uuid1().hex collection_id = 1001 archive_priority = create_priority() timestamp = create_timestamp() key = self._key_generator.next() segment_num = 4 sequence_num = 0 file_adler32 = zlib.adler32(test_data) file_md5 = hashlib.md5(test_data) slice_start = 0 slice_end = slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) unified_id_factory = UnifiedIDFactory(1) unified_id = unified_id_factory.next() message = { "message-type" : "archive-key-start", "priority" : archive_priority, "user-request-id" : user_request_id, "collection-id" : collection_id, "key" : key, "unified-id" : unified_id, "timestamp-repr" : repr(timestamp), "conjoined-part" : 0, "segment-num" : segment_num, "segment-size" : len(test_data[slice_start:slice_end]), "zfec-padding-size" : 4, "segment-md5-digest": b64encode(segment_md5.digest()), "segment-adler32" : segment_adler32, "sequence-num" : sequence_num, "source-node-name" : _local_node_name, "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end] ) self.assertEqual(reply["message-type"], "archive-key-start-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"]) for _ in range(slice_count-2): sequence_num += 1 slice_start += slice_size slice_end += slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type" : "archive-key-next", "priority" : archive_priority, "user-request-id" : user_request_id, "collection-id" : collection_id, "key" : key, "unified-id" : unified_id, "timestamp-repr" : repr(timestamp), "conjoined-part" : 0, "segment-num" : segment_num, "segment-size" : len(test_data[slice_start:slice_end]), "zfec-padding-size" : 4, "segment-md5-digest": b64encode(segment_md5.digest()), "segment-adler32" : segment_adler32, "sequence-num" : sequence_num, "source-node-name" : _local_node_name, "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end] ) self.assertEqual(reply["message-type"], "archive-key-next-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"]) sequence_num += 1 slice_start += slice_size slice_end += slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message = { "message-type" : "archive-key-final", "priority" : archive_priority, "user-request-id" : user_request_id, "collection-id" : collection_id, "key" : key, "unified-id" : unified_id, "timestamp-repr" : repr(timestamp), "conjoined-part" : 0, "segment-num" : segment_num, "segment-size" : len(test_data[slice_start:slice_end]), "zfec-padding-size" : 4, "segment-md5-digest": b64encode(segment_md5.digest()), "segment-adler32" : segment_adler32, "sequence-num" : sequence_num, "file-size" : total_size, "file-adler32" : file_adler32, "file-hash" : b64encode(file_md5.digest()), "source-node-name" : _local_node_name, "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end] ) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"])
def xxxtest_large_archive(self): """ test archiving a file that needs more than one message. For example, a 10 Mb file: each node would get 10 120kb zefec shares. """ slice_size = 1024 * 1024 slice_count = 10 total_size = slice_size * slice_count test_data = random_string(total_size) user_request_id = uuid.uuid1().hex collection_id = 1001 archive_priority = create_priority() timestamp = create_timestamp() key = self._key_generator.next() segment_num = 4 sequence_num = 0 file_adler32 = zlib.adler32(test_data) file_md5 = hashlib.md5(test_data) slice_start = 0 slice_end = slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) unified_id_factory = UnifiedIDFactory(1) unified_id = unified_id_factory.next() message = { "message-type": "archive-key-start", "priority": archive_priority, "user-request-id": user_request_id, "collection-id": collection_id, "key": key, "unified-id": unified_id, "timestamp-repr": repr(timestamp), "conjoined-part": 0, "segment-num": segment_num, "segment-size": len(test_data[slice_start:slice_end]), "zfec-padding-size": 4, "segment-md5-digest": b64encode(segment_md5.digest()), "segment-adler32": segment_adler32, "sequence-num": sequence_num, "source-node-name": _local_node_name, "handoff-node-name": None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end]) self.assertEqual(reply["message-type"], "archive-key-start-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"]) for _ in range(slice_count - 2): sequence_num += 1 slice_start += slice_size slice_end += slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type": "archive-key-next", "priority": archive_priority, "user-request-id": user_request_id, "collection-id": collection_id, "key": key, "unified-id": unified_id, "timestamp-repr": repr(timestamp), "conjoined-part": 0, "segment-num": segment_num, "segment-size": len(test_data[slice_start:slice_end]), "zfec-padding-size": 4, "segment-md5-digest": b64encode(segment_md5.digest()), "segment-adler32": segment_adler32, "sequence-num": sequence_num, "source-node-name": _local_node_name, "handoff-node-name": None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end]) self.assertEqual(reply["message-type"], "archive-key-next-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"]) sequence_num += 1 slice_start += slice_size slice_end += slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message = { "message-type": "archive-key-final", "priority": archive_priority, "user-request-id": user_request_id, "collection-id": collection_id, "key": key, "unified-id": unified_id, "timestamp-repr": repr(timestamp), "conjoined-part": 0, "segment-num": segment_num, "segment-size": len(test_data[slice_start:slice_end]), "zfec-padding-size": 4, "segment-md5-digest": b64encode(segment_md5.digest()), "segment-adler32": segment_adler32, "sequence-num": sequence_num, "file-size": total_size, "file-adler32": file_adler32, "file-hash": b64encode(file_md5.digest()), "source-node-name": _local_node_name, "handoff-node-name": None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end]) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["user-request-id"], user_request_id) self.assertEqual(reply["result"], "success", reply["error-message"])
def test_retrieve_large_content(self): """test retrieving content that fits in a multiple messages""" slice_size = 1024 * 1024 slice_count = 10 total_size = slice_size * slice_count test_data = random_string(total_size) collection_id = 1001 archive_priority = create_priority() timestamp = create_timestamp() key = self._key_generator.next() segment_num = 4 sequence_num = 0 file_adler32 = zlib.adler32(test_data) file_md5 = hashlib.md5(test_data) slice_start = 0 slice_end = slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type" : "archive-key-start", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "conjoined-unified-id" : None, "conjoined-part" : 0, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, "segment-size" : len(test_data[slice_start:slice_end]), "segment-adler32" : segment_adler32, "segment-md5-digest" : b64encode(segment_md5.digest()), "sequence-num" : sequence_num, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end] ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-start-reply") self.assertEqual(reply["result"], "success") for _ in range(slice_count-2): sequence_num += 1 slice_start += slice_size slice_end += slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type" : "archive-key-next", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "conjoined-unified-id" : None, "conjoined-part" : 0, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, "segment-size" : len( test_data[slice_start:slice_end] ), "segment-adler32" : segment_adler32, "segment-md5-digest" : b64encode(segment_md5.digest()), "sequence-num" : sequence_num, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end] ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-next-reply") self.assertEqual(reply["result"], "success") sequence_num += 1 slice_start += slice_size slice_end += slice_size self.assertEqual(slice_end, total_size) segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type" : "archive-key-final", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "conjoined-unified-id" : None, "conjoined-part" : 0, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, "segment-size" : len(test_data[slice_start:slice_end]), "segment-adler32" : segment_adler32, "segment-md5-digest" : b64encode(segment_md5.digest()), "sequence-num" : sequence_num, "file-size" : total_size, "file-adler32" : file_adler32, "file-hash" : b64encode(file_md5.digest()), "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end] ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success") # get file info from the local database _conjoined_row, segment_rows = current_status_of_key( self._database_connection, collection_id, key ) self.assertEqual(len(segment_rows), 1) retrieved_data_list = list() message_id = uuid.uuid1().hex message = { "message-type" : "retrieve-key-start", "message-id" : message_id, "collection-id" : collection_id, "key" : key, "timestamp-repr" : repr(timestamp), "conjoined-unified-id" : None, "conjoined-part" : 0, "segment-num" : segment_num } reply, data = send_request_and_get_reply_and_data( _local_node_name, _data_reader_address, _local_node_name, _client_address, message ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "retrieve-key-reply") self.assertEqual(reply["completed"], False) print "sequence-num =", reply["sequence-num"] retrieved_data_list.append(data) while True: message_id = uuid.uuid1().hex message = { "message-type" : "retrieve-key-next", "message-id" : message_id, "collection-id" : collection_id, "key" : key, "timestamp-repr" : repr(timestamp), "conjoined-unified-id" : None, "conjoined-part" : 0, "segment-num" : segment_num } reply, data = send_request_and_get_reply_and_data( _local_node_name, _data_reader_address, _local_node_name, _client_address, message ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "retrieve-key-reply") retrieved_data_list.append(data) print "sequence-num =", reply["sequence-num"] if reply["completed"]: break retrieved_data = "".join(retrieved_data_list) self.assertEqual(len(retrieved_data), len(test_data)) self.assertEqual(retrieved_data, test_data)
def test_retrieve_small_content(self): """test retrieving content that fits in a single message""" file_size = 10 * 64 * 1024 file_content = random_string(file_size) collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() timestamp = create_timestamp() segment_num = 2 file_adler32 = zlib.adler32(file_content) file_md5 = hashlib.md5(file_content) message_id = uuid.uuid1().hex message = { "message-type" : "archive-key-entire", "message-id" : message_id, "priority" : archive_priority, "collection-id" : collection_id, "key" : key, "conjoined-unified-id" : None, "conjoined-part" : 0, "timestamp-repr" : repr(timestamp), "segment-num" : segment_num, "segment-size" : file_size, "segment-adler32" : file_adler32, "segment-md5-digest" : b64encode(file_md5.digest()), "file-size" : file_size, "file-adler32" : file_adler32, "file-hash" : b64encode(file_md5.digest()), "handoff-node-name" : None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=file_content ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success") # get file info from the local database _conjoined_row, segment_rows = current_status_of_key( self._database_connection, collection_id, key ) self.assertEqual(len(segment_rows), 1) message_id = uuid.uuid1().hex message = { "message-type" : "retrieve-key-start", "message-id" : message_id, "collection-id" : collection_id, "key" : key, "timestamp-repr" : repr(timestamp), "conjoined-unified-id" : None, "conjoined-part" : 0, "segment-num" : segment_num } reply, data = send_request_and_get_reply_and_data( _local_node_name, _data_reader_address, _local_node_name, _client_address, message ) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "retrieve-key-reply") self.assertEqual(reply["completed"], True) self.assertEqual(len(data), len(file_content)) self.assertEqual(data, file_content)
def test_retrieve_small_content(self): """test retrieving content that fits in a single message""" file_size = 10 * 64 * 1024 file_content = random_string(file_size) collection_id = 1001 key = self._key_generator.next() archive_priority = create_priority() timestamp = create_timestamp() segment_num = 2 file_adler32 = zlib.adler32(file_content) file_md5 = hashlib.md5(file_content) message_id = uuid.uuid1().hex message = { "message-type": "archive-key-entire", "message-id": message_id, "priority": archive_priority, "collection-id": collection_id, "key": key, "conjoined-unified-id": None, "conjoined-part": 0, "timestamp-repr": repr(timestamp), "segment-num": segment_num, "segment-size": file_size, "segment-adler32": file_adler32, "segment-md5-digest": b64encode(file_md5.digest()), "file-size": file_size, "file-adler32": file_adler32, "file-hash": b64encode(file_md5.digest()), "handoff-node-name": None, } reply = send_request_and_get_reply(_local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=file_content) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success") # get file info from the local database _conjoined_row, segment_rows = current_status_of_key( self._database_connection, collection_id, key) self.assertEqual(len(segment_rows), 1) message_id = uuid.uuid1().hex message = { "message-type": "retrieve-key-start", "message-id": message_id, "collection-id": collection_id, "key": key, "timestamp-repr": repr(timestamp), "conjoined-unified-id": None, "conjoined-part": 0, "segment-num": segment_num } reply, data = send_request_and_get_reply_and_data( _local_node_name, _data_reader_address, _local_node_name, _client_address, message) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "retrieve-key-reply") self.assertEqual(reply["completed"], True) self.assertEqual(len(data), len(file_content)) self.assertEqual(data, file_content)
def test_retrieve_large_content(self): """test retrieving content that fits in a multiple messages""" slice_size = 1024 * 1024 slice_count = 10 total_size = slice_size * slice_count test_data = random_string(total_size) collection_id = 1001 archive_priority = create_priority() timestamp = create_timestamp() key = self._key_generator.next() segment_num = 4 sequence_num = 0 file_adler32 = zlib.adler32(test_data) file_md5 = hashlib.md5(test_data) slice_start = 0 slice_end = slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type": "archive-key-start", "message-id": message_id, "priority": archive_priority, "collection-id": collection_id, "key": key, "conjoined-unified-id": None, "conjoined-part": 0, "timestamp-repr": repr(timestamp), "segment-num": segment_num, "segment-size": len(test_data[slice_start:slice_end]), "segment-adler32": segment_adler32, "segment-md5-digest": b64encode(segment_md5.digest()), "sequence-num": sequence_num, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end]) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-start-reply") self.assertEqual(reply["result"], "success") for _ in range(slice_count - 2): sequence_num += 1 slice_start += slice_size slice_end += slice_size segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type": "archive-key-next", "message-id": message_id, "priority": archive_priority, "collection-id": collection_id, "key": key, "conjoined-unified-id": None, "conjoined-part": 0, "timestamp-repr": repr(timestamp), "segment-num": segment_num, "segment-size": len(test_data[slice_start:slice_end]), "segment-adler32": segment_adler32, "segment-md5-digest": b64encode(segment_md5.digest()), "sequence-num": sequence_num, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end]) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-next-reply") self.assertEqual(reply["result"], "success") sequence_num += 1 slice_start += slice_size slice_end += slice_size self.assertEqual(slice_end, total_size) segment_adler32 = zlib.adler32(test_data[slice_start:slice_end]) segment_md5 = hashlib.md5(test_data[slice_start:slice_end]) message_id = uuid.uuid1().hex message = { "message-type": "archive-key-final", "message-id": message_id, "priority": archive_priority, "collection-id": collection_id, "key": key, "conjoined-unified-id": None, "conjoined-part": 0, "timestamp-repr": repr(timestamp), "segment-num": segment_num, "segment-size": len(test_data[slice_start:slice_end]), "segment-adler32": segment_adler32, "segment-md5-digest": b64encode(segment_md5.digest()), "sequence-num": sequence_num, "file-size": total_size, "file-adler32": file_adler32, "file-hash": b64encode(file_md5.digest()), "handoff-node-name": None, } reply = send_request_and_get_reply( _local_node_name, _data_writer_address, _local_node_name, _client_address, message, data=test_data[slice_start:slice_end]) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "archive-key-final-reply") self.assertEqual(reply["result"], "success") # get file info from the local database _conjoined_row, segment_rows = current_status_of_key( self._database_connection, collection_id, key) self.assertEqual(len(segment_rows), 1) retrieved_data_list = list() message_id = uuid.uuid1().hex message = { "message-type": "retrieve-key-start", "message-id": message_id, "collection-id": collection_id, "key": key, "timestamp-repr": repr(timestamp), "conjoined-unified-id": None, "conjoined-part": 0, "segment-num": segment_num } reply, data = send_request_and_get_reply_and_data( _local_node_name, _data_reader_address, _local_node_name, _client_address, message) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "retrieve-key-reply") self.assertEqual(reply["completed"], False) print "sequence-num =", reply["sequence-num"] retrieved_data_list.append(data) while True: message_id = uuid.uuid1().hex message = { "message-type": "retrieve-key-next", "message-id": message_id, "collection-id": collection_id, "key": key, "timestamp-repr": repr(timestamp), "conjoined-unified-id": None, "conjoined-part": 0, "segment-num": segment_num } reply, data = send_request_and_get_reply_and_data( _local_node_name, _data_reader_address, _local_node_name, _client_address, message) self.assertEqual(reply["message-id"], message_id) self.assertEqual(reply["message-type"], "retrieve-key-reply") retrieved_data_list.append(data) print "sequence-num =", reply["sequence-num"] if reply["completed"]: break retrieved_data = "".join(retrieved_data_list) self.assertEqual(len(retrieved_data), len(test_data)) self.assertEqual(retrieved_data, test_data)