def test_header_serialization(self):
node = Node(node_name="test node",
pub_addr="10.10.10.10",
pub_port=1023)
task_header = TaskHeader(
task_id="xyz", environment="DEFAULT", task_owner=node)
# ignore dynamic properties
task_header.fixed_header.last_checking = 0
task_header_dict = task_header.to_dict()
serialized = CBORSerializer.dumps(task_header_dict)
deserialized = CBORSerializer.loads(serialized)
task_header_from_dict = TaskHeader.from_dict(deserialized)
# ignore dynamic properties
task_header_from_dict.fixed_header.last_checking = 0
assert task_header_from_dict.to_dict() == task_header_dict
assert isinstance(task_header_from_dict.task_owner, Node)
task_header_bin = task_header.to_binary()
bin_serialized = CBORSerializer.dumps(task_header_bin)
bin_deserialized = CBORSerializer.loads(bin_serialized)
assert bin_deserialized == task_header_bin
def get_short_hash(self):
"""Return short message representation for signature
:return str: short hash of serialized and sorted message dictionary
representation
"""
sorted_dict = self._sort_obj(self.dict_repr())
return SimpleHash.hash(CBORSerializer.dumps(sorted_dict))
def extract(self, input_path, output_dir=None, **kwargs):
files, files_dir = self.parent.extract(input_path,
output_dir=output_dir)
descriptor_path = os.path.join(files_dir, self.descriptor_file_name)
try:
with open(descriptor_path, 'rb') as src:
descriptor = CBORSerializer.loads(src.read())
os.remove(descriptor_path)
except Exception as e:
raise ValueError('Invalid package descriptor %r' % e.message)
if self.descriptor_file_name in files:
files.remove(self.descriptor_file_name)
if self.result_file_name in files:
files.remove(self.result_file_name)
extracted = ExtractedPackage(files, files_dir, descriptor)
if descriptor.result_type == result_types['data']:
result_path = os.path.join(files_dir, self.result_file_name)
with open(result_path, 'rb') as src:
extracted.result = src.read()
os.remove(result_path)
return extracted
def deserialize_message(cls, msg_):
"""
Deserialize single message
:param str msg_: serialized message
:return Message|None: deserialized message or none if this message
type is unknown
"""
try:
msg_repr = CBORSerializer.loads(msg_)
except Exception as exc:
logger.error("Error deserializing message: {}".format(exc))
msg_repr = None
logger.debug('msg_repr: %r', msg_repr)
if not (isinstance(msg_repr, list) and len(msg_repr) >= 4):
logger.info('Invalid message representation: %r', msg_repr)
return
msg_type = msg_repr[0]
msg_sig = msg_repr[1]
msg_timestamp = msg_repr[2]
d_repr = msg_repr[3]
if msg_type not in cls.registered_message_types:
logger.info('Unrecognized message type: %r', msg_type)
return
return cls.registered_message_types[msg_type](sig=msg_sig,
timestamp=msg_timestamp,
dict_repr=d_repr)
def serialize(self):
""" Return serialized message
:return str: serialized message """
try:
return CBORSerializer.dumps(
[self.TYPE, self.sig, self.timestamp,
self.dict_repr()])
except Exception:
logger.exception("Error serializing message:")
raise
def dict_to_binary(cls, dictionary):
self_dict = dict(dictionary)
self_dict.pop('last_checking', None)
self_dict.pop('signature', None)
self_dict['task_owner'] = cls._ordered(self_dict['task_owner'])
if self_dict.get('docker_images'):
self_dict['docker_images'] = [
cls._ordered(di) for di in self_dict['docker_images']
]
return CBORSerializer.dumps(cls._ordered(self_dict))
def test_header_serialization(self):
node = dict(node_name="test node")
docker_images = [
dict(repository="repo_{}".format(i),
id="id_{}".format(i),
tag="tag_{}".format(i)) for i in xrange(4)
]
task_header = TaskHeader("ABC",
"xyz",
"10.10.10.10",
1023,
"key",
"DEFAULT",
task_owner=node,
docker_images=docker_images)
# ignore dynamic properties
task_header.last_checking = 0
task_header_dict = task_header.to_dict()
serialized = CBORSerializer.dumps(task_header_dict)
deserialized = CBORSerializer.loads(serialized)
task_header_from_dict = TaskHeader.from_dict(deserialized)
# ignore dynamic properties
task_header_from_dict.last_checking = 0
assert task_header_from_dict.to_dict() == task_header_dict
assert isinstance(task_header_from_dict.task_owner, Node)
assert all([
isinstance(di, DockerImage)
for di in task_header_from_dict.docker_images
])
task_header_bin = task_header.to_binary()
bin_serialized = CBORSerializer.dumps(task_header_bin)
bin_deserialized = CBORSerializer.loads(bin_serialized)
assert bin_deserialized == task_header_bin
def __send_delta_resource(self, msg):
res_file_path = self.task_manager.get_resources(
msg.task_id, CBORSerializer.loads(msg.resource_header),
resource_types["zip"])
if not res_file_path:
logger.error("Task {} has no resource".format(msg.task_id))
self.conn.transport.write(struct.pack("!L", 0))
self.dropped()
return
self.conn.producer = tcpnetwork.EncryptFileProducer([res_file_path],
self)
def __send_resource_parts_list(self, msg):
res = self.task_manager.get_resources(
msg.task_id, CBORSerializer.loads(msg.resource_header),
resource_types["parts"])
if res is None:
return
delta_header, parts_list = res
self.send(
message.MessageDeltaParts(
task_id=self.task_id,
delta_header=delta_header,
parts=parts_list,
node_name=self.task_server.get_node_name(),
node_info=self.task_server.node,
address=self.task_server.get_resource_addr(),
port=self.task_server.get_resource_port()))
def create(self, output_path, disk_files=None, cbor_files=None, **kwargs):
if not disk_files and not cbor_files:
raise ValueError('No files to pack')
with self.generator(output_path) as of:
if disk_files:
for file_path in disk_files:
file_name = os.path.basename(file_path)
self.write_disk_file(of, file_path, file_name)
if cbor_files:
for file_name, file_data in cbor_files:
cbor_data = CBORSerializer.dumps(file_data)
self.write_cbor_file(of, file_name, cbor_data)
return output_path
def result_received(self, extra_data, decrypt=True):
""" Inform server about received result
:param dict extra_data: dictionary with information about
received result
:param bool decrypt: tells whether result decryption should
be performed
"""
result = extra_data.get('result')
result_type = extra_data.get("result_type")
subtask_id = extra_data.get("subtask_id")
if not subtask_id:
logger.error("No task_id value in extra_data for received data ")
return
if result_type is None:
logger.error("No information about result_type for received data ")
self._reject_subtask_result(subtask_id)
return
if result_type == result_types['data']:
try:
if decrypt:
result = self.decrypt(result)
result = CBORSerializer.loads(result)
except Exception as err:
logger.error("Can't load result data {}".format(err))
self._reject_subtask_result(subtask_id)
return
self.task_manager.computed_task_received(subtask_id, result,
result_type)
if not self.task_manager.verify_subtask(subtask_id):
self._reject_subtask_result(subtask_id)
return
self.task_server.accept_result(subtask_id, self.result_owner)
self.send(message.MessageSubtaskResultAccepted(subtask_id=subtask_id))
def dict_to_binary(cls, dictionary: dict) -> bytes:
return CBORSerializer.dumps(cls.dict_to_binarizable(dictionary))
def __send_data_results(self, res):
result = CBORSerializer.dumps(res.result)
extra_data = {"subtask_id": res.subtask_id, "data_type": "result"}
self.conn.producer = tcpnetwork.EncryptDataProducer(
self.encrypt(result), self, extra_data=extra_data)
def __compress_and_dump_file(file_name, data):
file_data = zlib.compress(data, 9)
return CBORSerializer.dumps((os.path.basename(file_name), file_data))
def test(self):
obj = MockSerializationSubject()
serialized = CBORSerializer.dumps(obj)
deserialized = CBORSerializer.loads(serialized)
assert_properties(deserialized, obj)
def _unpack_task_result(self, trp, output_dir):
tr = CBORSerializer.loads(trp)
with open(os.path.join(output_dir, tr[0]), "wb") as fh:
fh.write(decompress(tr[1]))
return os.path.join(output_dir, tr[0])
