def test_header_serialization(self):
node = Node(node_name="test node",
pub_addr="10.10.10.10",
pub_port=1023)
task_header = TaskHeader(
task_id="xyz", environment="DEFAULT", task_owner=node)
# ignore dynamic properties
task_header.fixed_header.last_checking = 0
task_header_dict = task_header.to_dict()
serialized = CBORSerializer.dumps(task_header_dict)
deserialized = CBORSerializer.loads(serialized)
task_header_from_dict = TaskHeader.from_dict(deserialized)
# ignore dynamic properties
task_header_from_dict.fixed_header.last_checking = 0
assert task_header_from_dict.to_dict() == task_header_dict
assert isinstance(task_header_from_dict.task_owner, Node)
task_header_bin = task_header.to_binary()
bin_serialized = CBORSerializer.dumps(task_header_bin)
bin_deserialized = CBORSerializer.loads(bin_serialized)
assert bin_deserialized == task_header_bin
def extract(self, input_path, output_dir=None, **kwargs):
files, files_dir = self.parent.extract(input_path,
output_dir=output_dir)
descriptor_path = os.path.join(files_dir, self.descriptor_file_name)
try:
with open(descriptor_path, 'rb') as src:
descriptor = CBORSerializer.loads(src.read())
os.remove(descriptor_path)
except Exception as e:
raise ValueError('Invalid package descriptor %r' % e.message)
if self.descriptor_file_name in files:
files.remove(self.descriptor_file_name)
if self.result_file_name in files:
files.remove(self.result_file_name)
extracted = ExtractedPackage(files, files_dir, descriptor)
if descriptor.result_type == result_types['data']:
result_path = os.path.join(files_dir, self.result_file_name)
with open(result_path, 'rb') as src:
extracted.result = src.read()
os.remove(result_path)
return extracted
def deserialize_message(cls, msg_):
"""
Deserialize single message
:param str msg_: serialized message
:return Message|None: deserialized message or none if this message
type is unknown
"""
try:
msg_repr = CBORSerializer.loads(msg_)
except Exception as exc:
logger.error("Error deserializing message: {}".format(exc))
msg_repr = None
logger.debug('msg_repr: %r', msg_repr)
if not (isinstance(msg_repr, list) and len(msg_repr) >= 4):
logger.info('Invalid message representation: %r', msg_repr)
return
msg_type = msg_repr[0]
msg_sig = msg_repr[1]
msg_timestamp = msg_repr[2]
d_repr = msg_repr[3]
if msg_type not in cls.registered_message_types:
logger.info('Unrecognized message type: %r', msg_type)
return
return cls.registered_message_types[msg_type](sig=msg_sig,
timestamp=msg_timestamp,
dict_repr=d_repr)
def test_header_serialization(self):
node = dict(node_name="test node")
docker_images = [
dict(repository="repo_{}".format(i),
id="id_{}".format(i),
tag="tag_{}".format(i)) for i in xrange(4)
]
task_header = TaskHeader("ABC",
"xyz",
"10.10.10.10",
1023,
"key",
"DEFAULT",
task_owner=node,
docker_images=docker_images)
# ignore dynamic properties
task_header.last_checking = 0
task_header_dict = task_header.to_dict()
serialized = CBORSerializer.dumps(task_header_dict)
deserialized = CBORSerializer.loads(serialized)
task_header_from_dict = TaskHeader.from_dict(deserialized)
# ignore dynamic properties
task_header_from_dict.last_checking = 0
assert task_header_from_dict.to_dict() == task_header_dict
assert isinstance(task_header_from_dict.task_owner, Node)
assert all([
isinstance(di, DockerImage)
for di in task_header_from_dict.docker_images
])
task_header_bin = task_header.to_binary()
bin_serialized = CBORSerializer.dumps(task_header_bin)
bin_deserialized = CBORSerializer.loads(bin_serialized)
assert bin_deserialized == task_header_bin
def __send_delta_resource(self, msg):
res_file_path = self.task_manager.get_resources(
msg.task_id, CBORSerializer.loads(msg.resource_header),
resource_types["zip"])
if not res_file_path:
logger.error("Task {} has no resource".format(msg.task_id))
self.conn.transport.write(struct.pack("!L", 0))
self.dropped()
return
self.conn.producer = tcpnetwork.EncryptFileProducer([res_file_path],
self)
def __send_resource_parts_list(self, msg):
res = self.task_manager.get_resources(
msg.task_id, CBORSerializer.loads(msg.resource_header),
resource_types["parts"])
if res is None:
return
delta_header, parts_list = res
self.send(
message.MessageDeltaParts(
task_id=self.task_id,
delta_header=delta_header,
parts=parts_list,
node_name=self.task_server.get_node_name(),
node_info=self.task_server.node,
address=self.task_server.get_resource_addr(),
port=self.task_server.get_resource_port()))
def result_received(self, extra_data, decrypt=True):
""" Inform server about received result
:param dict extra_data: dictionary with information about
received result
:param bool decrypt: tells whether result decryption should
be performed
"""
result = extra_data.get('result')
result_type = extra_data.get("result_type")
subtask_id = extra_data.get("subtask_id")
if not subtask_id:
logger.error("No task_id value in extra_data for received data ")
return
if result_type is None:
logger.error("No information about result_type for received data ")
self._reject_subtask_result(subtask_id)
return
if result_type == result_types['data']:
try:
if decrypt:
result = self.decrypt(result)
result = CBORSerializer.loads(result)
except Exception as err:
logger.error("Can't load result data {}".format(err))
self._reject_subtask_result(subtask_id)
return
self.task_manager.computed_task_received(subtask_id, result,
result_type)
if not self.task_manager.verify_subtask(subtask_id):
self._reject_subtask_result(subtask_id)
return
self.task_server.accept_result(subtask_id, self.result_owner)
self.send(message.MessageSubtaskResultAccepted(subtask_id=subtask_id))
def test(self):
obj = MockSerializationSubject()
serialized = CBORSerializer.dumps(obj)
deserialized = CBORSerializer.loads(serialized)
assert_properties(deserialized, obj)
def _unpack_task_result(self, trp, output_dir):
tr = CBORSerializer.loads(trp)
with open(os.path.join(output_dir, tr[0]), "wb") as fh:
fh.write(decompress(tr[1]))
return os.path.join(output_dir, tr[0])