def _send_message_receive_reply(data_writers, message, error_tag):
log = logging.getLogger(error_tag)
sender_list = list()
message["priority"] = create_priority()
pending_group = gevent.pool.Group()
for data_writer in data_writers:
# send a copy of the message, so each one gets a separagte message-id
sender = MessageGreenlet(data_writer, message.copy())
sender_list.append(sender)
pending_group.start(sender)
pending_group.join(timeout=_conjoined_timeout)
for sender in sender_list:
if not sender.ready():
log.error("incomplete")
raise ConjoinedFailedError("%s incomplete" % (error_tag, ))
if not sender.successful():
try:
sender.get()
except Exception, instance:
log.exception("")
raise ConjoinedFailedError("%s %s" % (error_tag, instance, ))
reply = sender.get()
if reply["result"] != "success":
log.error("%s" % (reply, ))
raise ConjoinedFailedError("%s %s" % (
error_tag, reply["error-message"]))
def _send_message_receive_reply(data_writers, message, error_tag):
log = logging.getLogger(error_tag)
sender_list = list()
message["priority"] = create_priority()
pending_group = gevent.pool.Group()
for data_writer in data_writers:
# send a copy of the message, so each one gets a separagte message-id
sender = MessageGreenlet(data_writer, message.copy())
sender_list.append(sender)
pending_group.start(sender)
pending_group.join(timeout=_conjoined_timeout)
for sender in sender_list:
if not sender.ready():
log.error("incomplete")
raise ConjoinedFailedError("%s incomplete" % (error_tag, ))
if not sender.successful():
try:
sender.get()
except Exception, instance:
log.exception("")
raise ConjoinedFailedError("%s %s" % (
error_tag,
instance,
))
reply = sender.get()
if reply["result"] != "success":
log.error("%s" % (reply, ))
raise ConjoinedFailedError("%s %s" %
(error_tag, reply["error-message"]))
def test_handoff_small_content(self):
"""test retrieving content that fits in a single message"""
file_size = 10 * 64 * 1024
file_content = random_string(file_size)
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 5
file_adler32 = zlib.adler32(file_content)
file_md5 = hashlib.md5(file_content)
message = {
"message-type": "archive-key-entire",
"priority": archive_priority,
"collection-id": collection_id,
"key": key,
"timestamp-repr": repr(timestamp),
"segment-num": segment_num,
"file-size": file_size,
"file-adler32": file_adler32,
"file-hash": b64encode(file_md5.digest()),
"handoff-node-name": None,
}
g = gevent.spawn(self._send_message_get_reply, message, file_content)
g.join(timeout=10.0)
self.assertEqual(g.ready(), True)
reply = g.value
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
print >> sys.stderr, "archive successful"
print >> sys.stderr, "press [Enter] to continue"
raw_input()
def _destroy(self, collection_id, key, unified_id_to_delete, timestamp,
segment_num):
user_request_id = uuid.uuid1().hex
archive_priority = create_priority()
unified_id_factory = UnifiedIDFactory(1)
unified_id = unified_id_factory.next()
message = {
"message-type": "destroy-key",
"priority": archive_priority,
"user-request-id": user_request_id,
"collection-id": collection_id,
"key": key,
"unified-id-to-delete": unified_id_to_delete,
"unified-id": unified_id,
"timestamp-repr": repr(timestamp),
"segment-num": segment_num,
"source-node-name": _local_node_name,
"handoff-node-name": None,
}
reply = send_request_and_get_reply(_local_node_name,
_data_writer_address,
_local_node_name, _client_address,
message)
self.assertEqual(reply["message-type"], "destroy-key-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
return reply
def destroy_key(
self,
collection_id,
key,
unified_id_to_delete,
unified_id,
timestamp,
segment_num,
source_node_name
):
message = {
"message-type" : "destroy-key",
"priority" : create_priority(),
"collection-id" : collection_id,
"key" : key,
"unified-id-to-delete" : unified_id_to_delete,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"source-node-name" : source_node_name,
"handoff-node-name" : None,
}
delivery_channel = \
self._resilient_client.queue_message_for_send(message)
self._log.debug(
'%(message-type)s: '
'key = %(key)r '
'segment_num = %(segment-num)d' % message
)
reply, _data = delivery_channel.get()
if reply["result"] != "success":
self._log.error("failed: %s" % (reply, ))
raise DestroyFailedError(reply["error-message"])
def _destroy(self, collection_id, key, unified_id_to_delete, timestamp,
segment_num):
user_request_id = uuid.uuid1().hex
archive_priority = create_priority()
unified_id_factory = UnifiedIDFactory(1)
unified_id = unified_id_factory.next()
message = {
"message-type" : "destroy-key",
"priority" : archive_priority,
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id-to-delete" : unified_id_to_delete,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"source-node-name" : _local_node_name,
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message
)
self.assertEqual(reply["message-type"], "destroy-key-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
return reply
def test_handoff_small_content(self):
"""test retrieving content that fits in a single message"""
file_size = 10 * 64 * 1024
file_content = random_string(file_size)
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 5
file_adler32 = zlib.adler32(file_content)
file_md5 = hashlib.md5(file_content)
message = {
"message-type" : "archive-key-entire",
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"handoff-node-name" : None,
}
g = gevent.spawn(self._send_message_get_reply, message, file_content)
g.join(timeout=10.0)
self.assertEqual(g.ready(), True)
reply = g.value
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
print >> sys.stderr, "archive successful"
print >> sys.stderr, "press [Enter] to continue"
raw_input()
def destroy_key(
self,
collection_id,
key,
unified_id_to_delete,
unified_id,
timestamp,
segment_num,
source_node_name,
user_request_id
):
message = {
"message-type" : "destroy-key",
"priority" : create_priority(),
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id-to-delete" : unified_id_to_delete,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"source-node-name" : source_node_name,
"handoff-node-name" : None,
}
delivery_channel = \
self._resilient_client.queue_message_for_send(message)
self._log.debug("request {user-request-id}: {message-type}: " \
"key = {key} " \
"timestamp = {timestamp-repr} " \
"segment_num = {segment-num}".format(**message))
reply, _data = delivery_channel.get()
return reply
def xxxtest_archive_key_entire_with_meta(self):
"""
test archiving a key in a single message, including meta data
"""
file_size = 10 * 64 * 1024
content_item = random_string(file_size)
user_request_id = uuid.uuid1().hex
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 2
meta_key = "".join([nimbus_meta_prefix, "test_key"])
meta_value = "pork"
file_adler32 = zlib.adler32(content_item)
file_md5 = hashlib.md5(content_item)
unified_id_factory = UnifiedIDFactory(1)
unified_id = unified_id_factory.next()
message = {
"message-type" : "archive-key-entire",
"priority" : archive_priority,
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"conjoined-part" : 0,
"segment-num" : segment_num,
"segment-size" : file_size,
"zfec-padding-size" : 4,
"segment-adler32" : file_adler32,
"segment-md5-digest": b64encode(file_md5.digest()),
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"source-node-name" : _local_node_name,
"handoff-node-name" : None,
meta_key : meta_value
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=content_item
)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
def xxxtest_destroy_tombstone(self):
"""test destroying a key that has already been destroyed"""
file_size = 10 * 64 * 1024
content_item = random_string(file_size)
message_id = uuid.uuid1().hex
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
archive_timestamp = create_timestamp()
destroy_1_timestamp = archive_timestamp + timedelta(seconds=1)
destroy_2_timestamp = destroy_1_timestamp + timedelta(seconds=1)
segment_num = 2
file_adler32 = zlib.adler32(content_item)
file_md5 = hashlib.md5(content_item)
message = {
"message-type" : "archive-key-entire",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(archive_timestamp),
"segment-num" : segment_num,
"segment-size" : file_size,
"segment-adler32" : file_adler32,
"segment-md5-digest": b64encode(file_md5.digest()),
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=content_item
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
reply = self._destroy(
collection_id, key, destroy_1_timestamp, segment_num
)
self.assertEqual(reply["result"], "success", reply["error-message"])
reply = self._destroy(
collection_id, key, destroy_2_timestamp, segment_num
)
self.assertEqual(reply["result"], "success", reply["error-message"])
def xxxtest_archive_key_entire_with_meta(self):
"""
test archiving a key in a single message, including meta data
"""
file_size = 10 * 64 * 1024
content_item = random_string(file_size)
user_request_id = uuid.uuid1().hex
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 2
meta_key = "".join([nimbus_meta_prefix, "test_key"])
meta_value = "pork"
file_adler32 = zlib.adler32(content_item)
file_md5 = hashlib.md5(content_item)
unified_id_factory = UnifiedIDFactory(1)
unified_id = unified_id_factory.next()
message = {
"message-type": "archive-key-entire",
"priority": archive_priority,
"user-request-id": user_request_id,
"collection-id": collection_id,
"key": key,
"unified-id": unified_id,
"timestamp-repr": repr(timestamp),
"conjoined-part": 0,
"segment-num": segment_num,
"segment-size": file_size,
"zfec-padding-size": 4,
"segment-adler32": file_adler32,
"segment-md5-digest": b64encode(file_md5.digest()),
"file-size": file_size,
"file-adler32": file_adler32,
"file-hash": b64encode(file_md5.digest()),
"source-node-name": _local_node_name,
"handoff-node-name": None,
meta_key: meta_value
}
reply = send_request_and_get_reply(_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=content_item)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
def archive_key_entire(
self,
collection_id,
key,
unified_id,
timestamp,
conjoined_part,
meta_dict,
segment_num,
zfec_padding_size,
file_size,
file_adler32,
file_md5,
segment,
source_node_name,
):
segment_md5 = hashlib.md5()
segment_md5.update(segment)
message = {
"message-type" : "archive-key-entire",
"priority" : create_priority(),
"collection-id" : collection_id,
"key" : key,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"conjoined-part" : conjoined_part,
"segment-num" : segment_num,
"segment-size" : len(segment),
"zfec-padding-size" : zfec_padding_size,
"segment-md5-digest" : b64encode(segment_md5.digest()),
"segment-adler32" : zlib.adler32(segment),
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5),
"source-node-name" : source_node_name,
"handoff-node-name" : None,
}
message.update(meta_dict)
delivery_channel = self._resilient_client.queue_message_for_send(
message, data=segment
)
self._log.debug(
'%(message-type)s: '
'key = %(key)r '
'timestamp = %(timestamp-repr)r '
'segment_num = %(segment-num)d' % message
)
reply, _data = delivery_channel.get()
if reply["result"] != "success":
self._log.error("failed: %s" % (reply, ))
raise ArchiveFailedError(reply["error-message"])
def _send_archive_cancel(user_request_id, unified_id, conjoined_part, clients):
# message sent to data writers telling them to cancel the archive
for i, client in enumerate(clients):
if not client.connected:
continue
cancel_message = {
"message-type" : "archive-key-cancel",
"priority" : create_priority(),
"user-request-id" : user_request_id,
"unified-id" : unified_id,
"conjoined-part" : conjoined_part,
"segment-num" : i+1,
}
client.queue_message_for_broadcast(cancel_message)
def archive_key_entire(
self,
collection_id,
key,
unified_id,
timestamp,
conjoined_part,
meta_dict,
segment_num,
zfec_padding_size,
file_size,
file_adler32,
file_md5,
segment,
source_node_name,
user_request_id
):
segment_size, segment_adler32, segment_md5 = \
_segment_properties(segment)
message = {
"message-type" : "archive-key-entire",
"priority" : create_priority(),
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"conjoined-part" : conjoined_part,
"segment-num" : segment_num,
"segment-size" : segment_size,
"zfec-padding-size" : zfec_padding_size,
"segment-md5-digest" : b64encode(segment_md5.digest()),
"segment-adler32" : segment_adler32,
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5),
"source-node-name" : source_node_name,
"handoff-node-name" : None,
}
message.update(meta_dict)
delivery_channel = self._resilient_client.queue_message_for_send(
message, data=segment
)
self._log.debug("request {user-request-id}: {message-type}: " \
"key = {key} " \
"timestamp = {timestamp-repr} " \
"segment_num = {segment-num}".format(**message))
reply, _data = delivery_channel.get()
return reply
def _send_archive_cancel(user_request_id, unified_id, conjoined_part, clients):
# message sent to data writers telling them to cancel the archive
for i, client in enumerate(clients):
if not client.connected:
continue
cancel_message = {
"message-type": "archive-key-cancel",
"priority": create_priority(),
"user-request-id": user_request_id,
"unified-id": unified_id,
"conjoined-part": conjoined_part,
"segment-num": i + 1,
}
client.queue_message_for_broadcast(cancel_message)
def _handle_archive_reply(state, message, _data):
log = logging.getLogger("_handle_archive_reply")
#TODO: we need to squawk about this somehow
if message["result"] != "success":
error_message = "%s failed (%s) %s %s" % (
message["message-type"],
message["result"],
message["error-message"],
message,
)
log.error(error_message)
raise HandoffError(error_message)
result = state["forwarder"].send(message)
if result is not None:
state["forwarder"] = None
segment_row, source_node_names = result
description = "handoff complete %s %s %s %s" % (
segment_row.collection_id,
segment_row.key,
segment_row.timestamp,
segment_row.segment_num,
)
log.info(description)
state["event-push-client"].info(
"handoff-complete",
description,
backup_sources=source_node_names,
collection_id=segment_row.collection_id,
key=segment_row.key,
timestamp_repr=repr(segment_row.timestamp)
)
# purge the handoff source(s)
message = {
"message-type" : "purge-handoff-source",
"priority" : create_priority(),
"collection-id" : segment_row.collection_id,
"unified-id" : segment_row.unified_id,
"handoff-node-id" : segment_row.handoff_node_id
}
for source_node_name in source_node_names:
writer_client = state["writer-client-dict"][source_node_name]
writer_client.queue_message_for_send(message)
def xxxtest_destroy_tombstone(self):
"""test destroying a key that has already been destroyed"""
file_size = 10 * 64 * 1024
content_item = random_string(file_size)
message_id = uuid.uuid1().hex
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
archive_timestamp = create_timestamp()
destroy_1_timestamp = archive_timestamp + timedelta(seconds=1)
destroy_2_timestamp = destroy_1_timestamp + timedelta(seconds=1)
segment_num = 2
file_adler32 = zlib.adler32(content_item)
file_md5 = hashlib.md5(content_item)
message = {
"message-type": "archive-key-entire",
"message-id": message_id,
"priority": archive_priority,
"collection-id": collection_id,
"key": key,
"timestamp-repr": repr(archive_timestamp),
"segment-num": segment_num,
"segment-size": file_size,
"segment-adler32": file_adler32,
"segment-md5-digest": b64encode(file_md5.digest()),
"file-size": file_size,
"file-adler32": file_adler32,
"file-hash": b64encode(file_md5.digest()),
"handoff-node-name": None,
}
reply = send_request_and_get_reply(_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=content_item)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
reply = self._destroy(collection_id, key, destroy_1_timestamp,
segment_num)
self.assertEqual(reply["result"], "success", reply["error-message"])
reply = self._destroy(collection_id, key, destroy_2_timestamp,
segment_num)
self.assertEqual(reply["result"], "success", reply["error-message"])
def append(self, message_tuple):
"""
add a message to the queue
The message must be created by tools.data_definitions.message_format
It must have a message["priority"] entry
"""
try:
priority = message_tuple[0]["priority"]
except KeyError:
self._log.error("message lacks priority %s" % (message_tuple[0], ))
priority = create_priority()
heapq.heappush(
self._internal_queue,
(priority, self._counter.next(), message_tuple, )
)
def append(self, message_tuple):
"""
add a message to the queue
The message must be created by tools.data_definitions.message_format
It must have a message["priority"] entry
"""
try:
priority = message_tuple[0]["priority"]
except KeyError:
self._log.error("message lacks priority %s" % (message_tuple[0], ))
priority = create_priority()
heapq.heappush(self._internal_queue, (
priority,
self._counter.next(),
message_tuple,
))
def test_archive_key_entire(self):
"""test archiving all data for a key in a single message"""
file_size = 10 * 64 * 1024
content_item = random_string(file_size)
message_id = uuid.uuid1().hex
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 2
file_adler32 = zlib.adler32(content_item)
file_md5 = hashlib.md5(content_item)
message = {
"message-type" : "archive-key-entire",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"segment-size" : file_size,
"segment-adler32" : file_adler32,
"segment-md5-digest": b64encode(file_md5.digest()),
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=content_item
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
def _purge(self, collection_id, key, timestamp, segment_num):
message_id = uuid.uuid1().hex
archive_priority = create_priority()
message = {
"message-type" : "purge-key",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "purge-key-reply")
return reply
def test_retrieve_large_content(self):
"""test retrieving content that fits in a multiple messages"""
slice_size = 1024 * 1024
slice_count = 10
total_size = slice_size * slice_count
test_data = random_string(total_size)
collection_id = 1001
archive_priority = create_priority()
timestamp = create_timestamp()
key = self._key_generator.next()
segment_num = 4
sequence_num = 0
file_adler32 = zlib.adler32(test_data)
file_md5 = hashlib.md5(test_data)
slice_start = 0
slice_end = slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type" : "archive-key-start",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"segment-size" : len(test_data[slice_start:slice_end]),
"segment-adler32" : segment_adler32,
"segment-md5-digest" : b64encode(segment_md5.digest()),
"sequence-num" : sequence_num,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end]
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-start-reply")
self.assertEqual(reply["result"], "success")
for _ in range(slice_count-2):
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type" : "archive-key-next",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"segment-size" : len(
test_data[slice_start:slice_end]
),
"segment-adler32" : segment_adler32,
"segment-md5-digest" : b64encode(segment_md5.digest()),
"sequence-num" : sequence_num,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end]
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-next-reply")
self.assertEqual(reply["result"], "success")
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
self.assertEqual(slice_end, total_size)
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type" : "archive-key-final",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"segment-size" : len(test_data[slice_start:slice_end]),
"segment-adler32" : segment_adler32,
"segment-md5-digest" : b64encode(segment_md5.digest()),
"sequence-num" : sequence_num,
"file-size" : total_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end]
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
# get file info from the local database
_conjoined_row, segment_rows = current_status_of_key(
self._database_connection, collection_id, key
)
self.assertEqual(len(segment_rows), 1)
retrieved_data_list = list()
message_id = uuid.uuid1().hex
message = {
"message-type" : "retrieve-key-start",
"message-id" : message_id,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(timestamp),
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"segment-num" : segment_num
}
reply, data = send_request_and_get_reply_and_data(
_local_node_name,
_data_reader_address,
_local_node_name,
_client_address,
message
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "retrieve-key-reply")
self.assertEqual(reply["completed"], False)
print "sequence-num =", reply["sequence-num"]
retrieved_data_list.append(data)
while True:
message_id = uuid.uuid1().hex
message = {
"message-type" : "retrieve-key-next",
"message-id" : message_id,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(timestamp),
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"segment-num" : segment_num
}
reply, data = send_request_and_get_reply_and_data(
_local_node_name,
_data_reader_address,
_local_node_name,
_client_address,
message
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "retrieve-key-reply")
retrieved_data_list.append(data)
print "sequence-num =", reply["sequence-num"]
if reply["completed"]:
break
retrieved_data = "".join(retrieved_data_list)
self.assertEqual(len(retrieved_data), len(test_data))
self.assertEqual(retrieved_data, test_data)
def test_retrieve_small_content(self):
"""test retrieving content that fits in a single message"""
file_size = 10 * 64 * 1024
file_content = random_string(file_size)
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 2
file_adler32 = zlib.adler32(file_content)
file_md5 = hashlib.md5(file_content)
message_id = uuid.uuid1().hex
message = {
"message-type": "archive-key-entire",
"message-id": message_id,
"priority": archive_priority,
"collection-id": collection_id,
"key": key,
"conjoined-unified-id": None,
"conjoined-part": 0,
"timestamp-repr": repr(timestamp),
"segment-num": segment_num,
"segment-size": file_size,
"segment-adler32": file_adler32,
"segment-md5-digest": b64encode(file_md5.digest()),
"file-size": file_size,
"file-adler32": file_adler32,
"file-hash": b64encode(file_md5.digest()),
"handoff-node-name": None,
}
reply = send_request_and_get_reply(_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=file_content)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
# get file info from the local database
_conjoined_row, segment_rows = current_status_of_key(
self._database_connection, collection_id, key)
self.assertEqual(len(segment_rows), 1)
message_id = uuid.uuid1().hex
message = {
"message-type": "retrieve-key-start",
"message-id": message_id,
"collection-id": collection_id,
"key": key,
"timestamp-repr": repr(timestamp),
"conjoined-unified-id": None,
"conjoined-part": 0,
"segment-num": segment_num
}
reply, data = send_request_and_get_reply_and_data(
_local_node_name, _data_reader_address, _local_node_name,
_client_address, message)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "retrieve-key-reply")
self.assertEqual(reply["completed"], True)
self.assertEqual(len(data), len(file_content))
self.assertEqual(data, file_content)
def test_retrieve_large_content(self):
"""test retrieving content that fits in a multiple messages"""
slice_size = 1024 * 1024
slice_count = 10
total_size = slice_size * slice_count
test_data = random_string(total_size)
collection_id = 1001
archive_priority = create_priority()
timestamp = create_timestamp()
key = self._key_generator.next()
segment_num = 4
sequence_num = 0
file_adler32 = zlib.adler32(test_data)
file_md5 = hashlib.md5(test_data)
slice_start = 0
slice_end = slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type": "archive-key-start",
"message-id": message_id,
"priority": archive_priority,
"collection-id": collection_id,
"key": key,
"conjoined-unified-id": None,
"conjoined-part": 0,
"timestamp-repr": repr(timestamp),
"segment-num": segment_num,
"segment-size": len(test_data[slice_start:slice_end]),
"segment-adler32": segment_adler32,
"segment-md5-digest": b64encode(segment_md5.digest()),
"sequence-num": sequence_num,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end])
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-start-reply")
self.assertEqual(reply["result"], "success")
for _ in range(slice_count - 2):
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type": "archive-key-next",
"message-id": message_id,
"priority": archive_priority,
"collection-id": collection_id,
"key": key,
"conjoined-unified-id": None,
"conjoined-part": 0,
"timestamp-repr": repr(timestamp),
"segment-num": segment_num,
"segment-size": len(test_data[slice_start:slice_end]),
"segment-adler32": segment_adler32,
"segment-md5-digest": b64encode(segment_md5.digest()),
"sequence-num": sequence_num,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end])
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-next-reply")
self.assertEqual(reply["result"], "success")
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
self.assertEqual(slice_end, total_size)
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type": "archive-key-final",
"message-id": message_id,
"priority": archive_priority,
"collection-id": collection_id,
"key": key,
"conjoined-unified-id": None,
"conjoined-part": 0,
"timestamp-repr": repr(timestamp),
"segment-num": segment_num,
"segment-size": len(test_data[slice_start:slice_end]),
"segment-adler32": segment_adler32,
"segment-md5-digest": b64encode(segment_md5.digest()),
"sequence-num": sequence_num,
"file-size": total_size,
"file-adler32": file_adler32,
"file-hash": b64encode(file_md5.digest()),
"handoff-node-name": None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end])
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
# get file info from the local database
_conjoined_row, segment_rows = current_status_of_key(
self._database_connection, collection_id, key)
self.assertEqual(len(segment_rows), 1)
retrieved_data_list = list()
message_id = uuid.uuid1().hex
message = {
"message-type": "retrieve-key-start",
"message-id": message_id,
"collection-id": collection_id,
"key": key,
"timestamp-repr": repr(timestamp),
"conjoined-unified-id": None,
"conjoined-part": 0,
"segment-num": segment_num
}
reply, data = send_request_and_get_reply_and_data(
_local_node_name, _data_reader_address, _local_node_name,
_client_address, message)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "retrieve-key-reply")
self.assertEqual(reply["completed"], False)
print "sequence-num =", reply["sequence-num"]
retrieved_data_list.append(data)
while True:
message_id = uuid.uuid1().hex
message = {
"message-type": "retrieve-key-next",
"message-id": message_id,
"collection-id": collection_id,
"key": key,
"timestamp-repr": repr(timestamp),
"conjoined-unified-id": None,
"conjoined-part": 0,
"segment-num": segment_num
}
reply, data = send_request_and_get_reply_and_data(
_local_node_name, _data_reader_address, _local_node_name,
_client_address, message)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "retrieve-key-reply")
retrieved_data_list.append(data)
print "sequence-num =", reply["sequence-num"]
if reply["completed"]:
break
retrieved_data = "".join(retrieved_data_list)
self.assertEqual(len(retrieved_data), len(test_data))
self.assertEqual(retrieved_data, test_data)
def xxxtest_large_archive(self):
"""
test archiving a file that needs more than one message.
For example, a 10 Mb file: each node would get 10 120kb
zefec shares.
"""
slice_size = 1024 * 1024
slice_count = 10
total_size = slice_size * slice_count
test_data = random_string(total_size)
user_request_id = uuid.uuid1().hex
collection_id = 1001
archive_priority = create_priority()
timestamp = create_timestamp()
key = self._key_generator.next()
segment_num = 4
sequence_num = 0
file_adler32 = zlib.adler32(test_data)
file_md5 = hashlib.md5(test_data)
slice_start = 0
slice_end = slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
unified_id_factory = UnifiedIDFactory(1)
unified_id = unified_id_factory.next()
message = {
"message-type" : "archive-key-start",
"priority" : archive_priority,
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"conjoined-part" : 0,
"segment-num" : segment_num,
"segment-size" : len(test_data[slice_start:slice_end]),
"zfec-padding-size" : 4,
"segment-md5-digest": b64encode(segment_md5.digest()),
"segment-adler32" : segment_adler32,
"sequence-num" : sequence_num,
"source-node-name" : _local_node_name,
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end]
)
self.assertEqual(reply["message-type"], "archive-key-start-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
for _ in range(slice_count-2):
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type" : "archive-key-next",
"priority" : archive_priority,
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"conjoined-part" : 0,
"segment-num" : segment_num,
"segment-size" : len(test_data[slice_start:slice_end]),
"zfec-padding-size" : 4,
"segment-md5-digest": b64encode(segment_md5.digest()),
"segment-adler32" : segment_adler32,
"sequence-num" : sequence_num,
"source-node-name" : _local_node_name,
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end]
)
self.assertEqual(reply["message-type"], "archive-key-next-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message = {
"message-type" : "archive-key-final",
"priority" : archive_priority,
"user-request-id" : user_request_id,
"collection-id" : collection_id,
"key" : key,
"unified-id" : unified_id,
"timestamp-repr" : repr(timestamp),
"conjoined-part" : 0,
"segment-num" : segment_num,
"segment-size" : len(test_data[slice_start:slice_end]),
"zfec-padding-size" : 4,
"segment-md5-digest": b64encode(segment_md5.digest()),
"segment-adler32" : segment_adler32,
"sequence-num" : sequence_num,
"file-size" : total_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"source-node-name" : _local_node_name,
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end]
)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
def forwarder_coroutine(node_dict, segment_row, writer_socket, reader_socket):
"""
manage the message traffic for retrieving and re-archiving
a segment that was handed off to us
"""
log = logging.getLogger("forwarder_coroutine")
user_request_id = str(uuid.uuid4())
archive_priority = create_priority()
retrieve_id = uuid.uuid1().hex
retrieve_sequence = 0
# start retrieving from our reader
message = {
"message-type" : "retrieve-key-start",
"user-request-id" : user_request_id,
"retrieve-id" : retrieve_id,
"retrieve-sequence" : retrieve_sequence,
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"segment-unified-id" : segment_row["unified_id"],
"segment-conjoined-part" : segment_row["conjoined_part"],
"segment-num" : segment_row["segment_num"],
"handoff-node-id" : segment_row["handoff_node_id"],
"block-offset" : 0,
"block-count" : None,
}
log.debug("request {0}: " \
"sending retrieve-key-start {1} {2}".format(
user_request_id,
segment_row["unified_id"],
segment_row["segment_num"]))
reader_socket.send(message)
try:
reader_socket.wait_for_ack()
except ReqSocketAckTimeOut as instance:
log.error("request {0}: " \
"timeout waiting ack {1} {2}".format(user_request_id,
str(reader_socket),
str(instance)))
raise
reply, data = yield
assert reply["message-type"] == "retrieve-key-reply", reply
assert reply["result"] == "success", reply
completed = reply["completed"]
sequence = 1
if completed:
message = {
"message-type" : "archive-key-entire",
"user-request-id" : user_request_id,
"priority" : archive_priority,
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"unified-id" : segment_row["unified_id"],
"conjoined-part" : segment_row["conjoined_part"],
"timestamp-repr" : repr(segment_row["timestamp"]),
"segment-num" : segment_row["segment_num"],
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"file-size" : segment_row["file_size"],
"file-adler32" : segment_row["file_adler32"],
"file-hash" : segment_row["file_hash"],
"source-node-name" : node_dict[segment_row["source_node_id"]],
"handoff-node-name" : None,
}
else:
message = {
"message-type" : "archive-key-start",
"user-request-id" : user_request_id,
"priority" : archive_priority,
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"unified-id" : segment_row["unified_id"],
"conjoined-part" : segment_row["conjoined_part"],
"timestamp-repr" : repr(segment_row["timestamp"]),
"segment-num" : segment_row["segment_num"],
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"sequence-num" : sequence,
"source-node-name" : node_dict[segment_row["source_node_id"]],
"handoff-node-name" : None,
}
writer_socket.send(message, data=data)
try:
writer_socket.wait_for_ack()
except ReqSocketAckTimeOut as instance:
log.error("request {0}: " \
"timeout waiting ack {1} {2}".format(user_request_id,
str(writer_socket),
str(instance)))
raise
reply, _ = yield
if completed:
yield "done"
return
assert reply["message-type"] == "archive-key-start-reply", reply
assert reply["result"] == "success", reply
# send the intermediate segments
while not completed:
retrieve_sequence += 1
sequence += 1
message = {
"message-type" : "retrieve-key-next",
"user-request-id" : user_request_id,
"retrieve-id" : retrieve_id,
"retrieve-sequence" : retrieve_sequence,
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"segment-unified-id" : segment_row["unified_id"],
"segment-conjoined-part" : segment_row["conjoined_part"],
"segment-num" : segment_row["segment_num"],
"handoff-node-id" : segment_row["handoff_node_id"],
"block-offset" : 0,
"block-count" : None,
}
reader_socket.send(message)
try:
reader_socket.wait_for_ack()
except ReqSocketAckTimeOut as instance:
log.error("request {0}: " \
"timeout waiting ack {1} {2}".format(user_request_id,
str(reader_socket),
str(instance)))
raise
reply, data = yield
assert reply["message-type"] == "retrieve-key-reply", reply
assert reply["result"] == "success", reply
completed = reply["completed"]
if completed:
message = {
"message-type" : "archive-key-final",
"user-request-id" : user_request_id,
"priority" : archive_priority,
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"unified-id" : segment_row["unified_id"],
"conjoined-part" : segment_row["conjoined_part"],
"timestamp-repr" : repr(segment_row["timestamp"]),
"segment-num" : segment_row["segment_num"],
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"sequence-num" : sequence,
"file-size" : segment_row["file_size"],
"file-adler32" : segment_row["file_adler32"],
"file-hash" : segment_row["file_hash"],
"source-node-name" : node_dict[segment_row["source_node_id"]],
"handoff-node-name" : None,
}
else:
message = {
"message-type" : "archive-key-next",
"user-request-id" : user_request_id,
"priority" : archive_priority,
"collection-id" : segment_row["collection_id"],
"key" : segment_row["key"],
"unified-id" : segment_row["unified_id"],
"conjoined-part" : segment_row["conjoined_part"],
"timestamp-repr" : repr(segment_row["timestamp"]),
"segment-num" : segment_row["segment_num"],
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"sequence-num" : sequence,
"source-node-name" : node_dict[segment_row["source_node_id"]],
"handoff-node-name" : None,
}
writer_socket.send(message, data=data)
try:
writer_socket.wait_for_ack()
except ReqSocketAckTimeOut as instance:
log.error("request {0}: " \
"timeout waiting ack {1} {2}".format(user_request_id,
str(writer_socket),
str(instance)))
raise
reply, _ = yield
assert reply["result"] == "success", reply
yield "done"
def xxxtest_large_archive(self):
"""
test archiving a file that needs more than one message.
For example, a 10 Mb file: each node would get 10 120kb
zefec shares.
"""
slice_size = 1024 * 1024
slice_count = 10
total_size = slice_size * slice_count
test_data = random_string(total_size)
user_request_id = uuid.uuid1().hex
collection_id = 1001
archive_priority = create_priority()
timestamp = create_timestamp()
key = self._key_generator.next()
segment_num = 4
sequence_num = 0
file_adler32 = zlib.adler32(test_data)
file_md5 = hashlib.md5(test_data)
slice_start = 0
slice_end = slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
unified_id_factory = UnifiedIDFactory(1)
unified_id = unified_id_factory.next()
message = {
"message-type": "archive-key-start",
"priority": archive_priority,
"user-request-id": user_request_id,
"collection-id": collection_id,
"key": key,
"unified-id": unified_id,
"timestamp-repr": repr(timestamp),
"conjoined-part": 0,
"segment-num": segment_num,
"segment-size": len(test_data[slice_start:slice_end]),
"zfec-padding-size": 4,
"segment-md5-digest": b64encode(segment_md5.digest()),
"segment-adler32": segment_adler32,
"sequence-num": sequence_num,
"source-node-name": _local_node_name,
"handoff-node-name": None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end])
self.assertEqual(reply["message-type"], "archive-key-start-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
for _ in range(slice_count - 2):
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message_id = uuid.uuid1().hex
message = {
"message-type": "archive-key-next",
"priority": archive_priority,
"user-request-id": user_request_id,
"collection-id": collection_id,
"key": key,
"unified-id": unified_id,
"timestamp-repr": repr(timestamp),
"conjoined-part": 0,
"segment-num": segment_num,
"segment-size": len(test_data[slice_start:slice_end]),
"zfec-padding-size": 4,
"segment-md5-digest": b64encode(segment_md5.digest()),
"segment-adler32": segment_adler32,
"sequence-num": sequence_num,
"source-node-name": _local_node_name,
"handoff-node-name": None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end])
self.assertEqual(reply["message-type"], "archive-key-next-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success",
reply["error-message"])
sequence_num += 1
slice_start += slice_size
slice_end += slice_size
segment_adler32 = zlib.adler32(test_data[slice_start:slice_end])
segment_md5 = hashlib.md5(test_data[slice_start:slice_end])
message = {
"message-type": "archive-key-final",
"priority": archive_priority,
"user-request-id": user_request_id,
"collection-id": collection_id,
"key": key,
"unified-id": unified_id,
"timestamp-repr": repr(timestamp),
"conjoined-part": 0,
"segment-num": segment_num,
"segment-size": len(test_data[slice_start:slice_end]),
"zfec-padding-size": 4,
"segment-md5-digest": b64encode(segment_md5.digest()),
"segment-adler32": segment_adler32,
"sequence-num": sequence_num,
"file-size": total_size,
"file-adler32": file_adler32,
"file-hash": b64encode(file_md5.digest()),
"source-node-name": _local_node_name,
"handoff-node-name": None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=test_data[slice_start:slice_end])
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["user-request-id"], user_request_id)
self.assertEqual(reply["result"], "success", reply["error-message"])
def test_retrieve_small_content(self):
"""test retrieving content that fits in a single message"""
file_size = 10 * 64 * 1024
file_content = random_string(file_size)
collection_id = 1001
key = self._key_generator.next()
archive_priority = create_priority()
timestamp = create_timestamp()
segment_num = 2
file_adler32 = zlib.adler32(file_content)
file_md5 = hashlib.md5(file_content)
message_id = uuid.uuid1().hex
message = {
"message-type" : "archive-key-entire",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : collection_id,
"key" : key,
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"timestamp-repr" : repr(timestamp),
"segment-num" : segment_num,
"segment-size" : file_size,
"segment-adler32" : file_adler32,
"segment-md5-digest" : b64encode(file_md5.digest()),
"file-size" : file_size,
"file-adler32" : file_adler32,
"file-hash" : b64encode(file_md5.digest()),
"handoff-node-name" : None,
}
reply = send_request_and_get_reply(
_local_node_name,
_data_writer_address,
_local_node_name,
_client_address,
message,
data=file_content
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "archive-key-final-reply")
self.assertEqual(reply["result"], "success")
# get file info from the local database
_conjoined_row, segment_rows = current_status_of_key(
self._database_connection, collection_id, key
)
self.assertEqual(len(segment_rows), 1)
message_id = uuid.uuid1().hex
message = {
"message-type" : "retrieve-key-start",
"message-id" : message_id,
"collection-id" : collection_id,
"key" : key,
"timestamp-repr" : repr(timestamp),
"conjoined-unified-id" : None,
"conjoined-part" : 0,
"segment-num" : segment_num
}
reply, data = send_request_and_get_reply_and_data(
_local_node_name,
_data_reader_address,
_local_node_name,
_client_address,
message
)
self.assertEqual(reply["message-id"], message_id)
self.assertEqual(reply["message-type"], "retrieve-key-reply")
self.assertEqual(reply["completed"], True)
self.assertEqual(len(data), len(file_content))
self.assertEqual(data, file_content)
def forwarder_coroutine(
node_name_dict,
segment_row,
source_node_names,
writer_client,
reader_client
):
"""
manage the message traffic for retrieving and re-archiving
a segment that was handed off to us
"""
log = logging.getLogger("forwarder_coroutine")
archive_priority = create_priority()
# start retrieving from our reader
message_id = uuid.uuid1().hex
message = {
"message-type" : "retrieve-key-start",
"message-id" : message_id,
"segment-unified-id": segment_row.unified_id,
"segment-num" : segment_row.segment_num,
}
log.debug("sending retrieve-key-start %s %s" % (
segment_row.unified_id,
segment_row.segment_num
))
reader_client.queue_message_for_send(message, data=None)
reply, data = yield
assert reply["message-type"] == "retrieve-key-reply", reply
assert reply["result"] == "success", reply
completed = reply["completed"]
sequence = 1
message_id = uuid.uuid1().hex
if completed:
message = {
"message-type" : "archive-key-entire",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : segment_row.collection_id,
"key" : segment_row.key,
"unified-id" : segment_row.unified_id,
"conjoined-part" : segment_row.conjoined_part,
"timestamp-repr" : repr(segment_row.timestamp),
"segment-num" : segment_row.segment_num,
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"file-size" : segment_row.file_size,
"file-adler32" : segment_row.file_adler32,
"file-hash" : b64encode(segment_row.file_hash),
"source-node-name" : node_name_dict[segment_row.source_node_id],
"handoff-node-name" : None,
}
else:
message = {
"message-type" : "archive-key-start",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : segment_row.collection_id,
"key" : segment_row.key,
"unified-id" : segment_row.unified_id,
"conjoined-part" : segment_row.conjoined_part,
"timestamp-repr" : repr(segment_row.timestamp),
"segment-num" : segment_row.segment_num,
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"sequence-num" : sequence,
"source-node-name" : node_name_dict[segment_row.source_node_id],
"handoff-node-name" : None,
}
writer_client.queue_message_for_send(message, data=data)
reply = yield
if completed:
# we give back the segment_row and source node names as our last yield
yield (segment_row, source_node_names, )
return
assert reply["message-type"] == "archive-key-start-reply", reply
assert reply["result"] == "success", reply
# send the intermediate segments
while not completed:
sequence += 1
message_id = uuid.uuid1().hex
message = {
"message-type" : "retrieve-key-next",
"message-id" : message_id,
"segment-unified-id": segment_row.unified_id,
"segment-num" : segment_row.segment_num,
}
reader_client.queue_message_for_send(message, data=None)
reply, data = yield
assert reply["message-type"] == "retrieve-key-reply", reply
assert reply["result"] == "success", reply
completed = reply["completed"]
message_id = uuid.uuid1().hex
if completed:
message = {
"message-type" : "archive-key-final",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : segment_row.collection_id,
"key" : segment_row.key,
"unified-id" : segment_row.unified_id,
"conjoined-part" : segment_row.conjoined_part,
"timestamp-repr" : repr(segment_row.timestamp),
"segment-num" : segment_row.segment_num,
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"sequence-num" : sequence,
"file-size" : segment_row.file_size,
"file-adler32" : segment_row.file_adler32,
"file-hash" : b64encode(segment_row.file_hash),
"source-node-name" : node_name_dict[
segment_row.source_node_id],
"handoff-node-name" : None,
}
else:
message = {
"message-type" : "archive-key-next",
"message-id" : message_id,
"priority" : archive_priority,
"collection-id" : segment_row.collection_id,
"key" : segment_row.key,
"unified-id" : segment_row.unified_id,
"conjoined-part" : segment_row.conjoined_part,
"timestamp-repr" : repr(segment_row.timestamp),
"segment-num" : segment_row.segment_num,
"segment-size" : reply["segment-size"],
"zfec-padding-size" : reply["zfec-padding-size"],
"segment-adler32" : reply["segment-adler32"],
"segment-md5-digest": reply["segment-md5-digest"],
"sequence-num" : sequence,
"source-node-name" : node_name_dict[
segment_row.source_node_id],
"handoff-node-name" : None,
}
writer_client.queue_message_for_send(message, data=data)
reply = yield
assert reply["result"] == "success", reply
# we give back the segment_row and source node names as our last yield
yield (segment_row, source_node_names, )
def __init__(self):
self._log = logging.getLogger("WebServer")
authenticator = SqlAuthenticator()
self._central_connection = get_central_connection()
self._cluster_row = get_cluster_row(self._central_connection)
self._node_local_connection = get_node_local_connection()
self._unified_id_factory = UnifiedIDFactory(
self._central_connection,
_get_shard_id(self._central_connection, self._cluster_row.id)
)
self._deliverator = Deliverator()
self._zeromq_context = zmq.Context()
self._pull_server = GreenletPULLServer(
self._zeromq_context,
_web_server_pipeline_address,
self._deliverator
)
self._pull_server.link_exception(self._unhandled_greenlet_exception)
# message sent to data readers and writers telling them the server
# is (re)starting, thereby invalidating any archvies or retrieved
# that are in progress for this node
timestamp = create_timestamp()
start_message = {
"message-type" : "web-server-start",
"priority" : create_priority(),
"unified-id" : self._unified_id_factory.next(),
"timestamp-repr" : repr(timestamp),
"source-node-name" : _local_node_name,
}
self._data_writer_clients = list()
for node_name, address in zip(_node_names, _data_writer_addresses):
resilient_client = GreenletResilientClient(
self._zeromq_context,
node_name,
address,
_client_tag,
_web_server_pipeline_address,
self._deliverator,
connect_messages=[start_message, ]
)
resilient_client.link_exception(self._unhandled_greenlet_exception)
self._data_writer_clients.append(resilient_client)
self._data_reader_clients = list()
self._data_readers = list()
for node_name, address in zip(_node_names, _data_reader_addresses):
resilient_client = GreenletResilientClient(
self._zeromq_context,
node_name,
address,
_client_tag,
_web_server_pipeline_address,
self._deliverator,
connect_messages=[start_message, ]
)
resilient_client.link_exception(self._unhandled_greenlet_exception)
self._data_reader_clients.append(resilient_client)
data_reader = DataReader(
node_name, resilient_client
)
self._data_readers.append(data_reader)
self._space_accounting_dealer_client = GreenletDealerClient(
self._zeromq_context,
_local_node_name,
_space_accounting_server_address
)
self._space_accounting_dealer_client.link_exception(
self._unhandled_greenlet_exception
)
push_client = GreenletPUSHClient(
self._zeromq_context,
_local_node_name,
_space_accounting_pipeline_address,
)
self._accounting_client = SpaceAccountingClient(
_local_node_name,
self._space_accounting_dealer_client,
push_client
)
self._event_push_client = EventPushClient(
self._zeromq_context,
"web-server"
)
self._watcher = Watcher(
_stats,
self._data_reader_clients,
self._data_writer_clients,
self._event_push_client
)
id_translator_keys_path = os.path.join(
_repository_path, "id_translator_keys.pkl"
)
with open(id_translator_keys_path, "r") as input_file:
id_translator_keys = pickle.load(input_file)
self._id_translator = InternalIDTranslator(
id_translator_keys["key"],
id_translator_keys["hmac_key"],
id_translator_keys["iv_key"],
id_translator_keys["hmac_size"]
)
self.application = Application(
self._central_connection,
self._node_local_connection,
self._cluster_row,
self._unified_id_factory,
self._id_translator,
self._data_writer_clients,
self._data_readers,
authenticator,
self._accounting_client,
self._event_push_client,
_stats
)
self.wsgi_server = WSGIServer(
(_web_server_host, _web_server_port),
application=self.application,
backlog=_wsgi_backlog
)
