def test_header_serialization(self): node = Node(node_name="test node", pub_addr="10.10.10.10", pub_port=1023) task_header = TaskHeader( task_id="xyz", environment="DEFAULT", task_owner=node) # ignore dynamic properties task_header.fixed_header.last_checking = 0 task_header_dict = task_header.to_dict() serialized = CBORSerializer.dumps(task_header_dict) deserialized = CBORSerializer.loads(serialized) task_header_from_dict = TaskHeader.from_dict(deserialized) # ignore dynamic properties task_header_from_dict.fixed_header.last_checking = 0 assert task_header_from_dict.to_dict() == task_header_dict assert isinstance(task_header_from_dict.task_owner, Node) task_header_bin = task_header.to_binary() bin_serialized = CBORSerializer.dumps(task_header_bin) bin_deserialized = CBORSerializer.loads(bin_serialized) assert bin_deserialized == task_header_bin
def get_short_hash(self): """Return short message representation for signature :return str: short hash of serialized and sorted message dictionary representation """ sorted_dict = self._sort_obj(self.dict_repr()) return SimpleHash.hash(CBORSerializer.dumps(sorted_dict))
def extract(self, input_path, output_dir=None, **kwargs): files, files_dir = self.parent.extract(input_path, output_dir=output_dir) descriptor_path = os.path.join(files_dir, self.descriptor_file_name) try: with open(descriptor_path, 'rb') as src: descriptor = CBORSerializer.loads(src.read()) os.remove(descriptor_path) except Exception as e: raise ValueError('Invalid package descriptor %r' % e.message) if self.descriptor_file_name in files: files.remove(self.descriptor_file_name) if self.result_file_name in files: files.remove(self.result_file_name) extracted = ExtractedPackage(files, files_dir, descriptor) if descriptor.result_type == result_types['data']: result_path = os.path.join(files_dir, self.result_file_name) with open(result_path, 'rb') as src: extracted.result = src.read() os.remove(result_path) return extracted
def deserialize_message(cls, msg_): """ Deserialize single message :param str msg_: serialized message :return Message|None: deserialized message or none if this message type is unknown """ try: msg_repr = CBORSerializer.loads(msg_) except Exception as exc: logger.error("Error deserializing message: {}".format(exc)) msg_repr = None logger.debug('msg_repr: %r', msg_repr) if not (isinstance(msg_repr, list) and len(msg_repr) >= 4): logger.info('Invalid message representation: %r', msg_repr) return msg_type = msg_repr[0] msg_sig = msg_repr[1] msg_timestamp = msg_repr[2] d_repr = msg_repr[3] if msg_type not in cls.registered_message_types: logger.info('Unrecognized message type: %r', msg_type) return return cls.registered_message_types[msg_type](sig=msg_sig, timestamp=msg_timestamp, dict_repr=d_repr)
def serialize(self): """ Return serialized message :return str: serialized message """ try: return CBORSerializer.dumps( [self.TYPE, self.sig, self.timestamp, self.dict_repr()]) except Exception: logger.exception("Error serializing message:") raise
def dict_to_binary(cls, dictionary): self_dict = dict(dictionary) self_dict.pop('last_checking', None) self_dict.pop('signature', None) self_dict['task_owner'] = cls._ordered(self_dict['task_owner']) if self_dict.get('docker_images'): self_dict['docker_images'] = [ cls._ordered(di) for di in self_dict['docker_images'] ] return CBORSerializer.dumps(cls._ordered(self_dict))
def test_header_serialization(self): node = dict(node_name="test node") docker_images = [ dict(repository="repo_{}".format(i), id="id_{}".format(i), tag="tag_{}".format(i)) for i in xrange(4) ] task_header = TaskHeader("ABC", "xyz", "10.10.10.10", 1023, "key", "DEFAULT", task_owner=node, docker_images=docker_images) # ignore dynamic properties task_header.last_checking = 0 task_header_dict = task_header.to_dict() serialized = CBORSerializer.dumps(task_header_dict) deserialized = CBORSerializer.loads(serialized) task_header_from_dict = TaskHeader.from_dict(deserialized) # ignore dynamic properties task_header_from_dict.last_checking = 0 assert task_header_from_dict.to_dict() == task_header_dict assert isinstance(task_header_from_dict.task_owner, Node) assert all([ isinstance(di, DockerImage) for di in task_header_from_dict.docker_images ]) task_header_bin = task_header.to_binary() bin_serialized = CBORSerializer.dumps(task_header_bin) bin_deserialized = CBORSerializer.loads(bin_serialized) assert bin_deserialized == task_header_bin
def __send_delta_resource(self, msg): res_file_path = self.task_manager.get_resources( msg.task_id, CBORSerializer.loads(msg.resource_header), resource_types["zip"]) if not res_file_path: logger.error("Task {} has no resource".format(msg.task_id)) self.conn.transport.write(struct.pack("!L", 0)) self.dropped() return self.conn.producer = tcpnetwork.EncryptFileProducer([res_file_path], self)
def __send_resource_parts_list(self, msg): res = self.task_manager.get_resources( msg.task_id, CBORSerializer.loads(msg.resource_header), resource_types["parts"]) if res is None: return delta_header, parts_list = res self.send( message.MessageDeltaParts( task_id=self.task_id, delta_header=delta_header, parts=parts_list, node_name=self.task_server.get_node_name(), node_info=self.task_server.node, address=self.task_server.get_resource_addr(), port=self.task_server.get_resource_port()))
def create(self, output_path, disk_files=None, cbor_files=None, **kwargs): if not disk_files and not cbor_files: raise ValueError('No files to pack') with self.generator(output_path) as of: if disk_files: for file_path in disk_files: file_name = os.path.basename(file_path) self.write_disk_file(of, file_path, file_name) if cbor_files: for file_name, file_data in cbor_files: cbor_data = CBORSerializer.dumps(file_data) self.write_cbor_file(of, file_name, cbor_data) return output_path
def result_received(self, extra_data, decrypt=True): """ Inform server about received result :param dict extra_data: dictionary with information about received result :param bool decrypt: tells whether result decryption should be performed """ result = extra_data.get('result') result_type = extra_data.get("result_type") subtask_id = extra_data.get("subtask_id") if not subtask_id: logger.error("No task_id value in extra_data for received data ") return if result_type is None: logger.error("No information about result_type for received data ") self._reject_subtask_result(subtask_id) return if result_type == result_types['data']: try: if decrypt: result = self.decrypt(result) result = CBORSerializer.loads(result) except Exception as err: logger.error("Can't load result data {}".format(err)) self._reject_subtask_result(subtask_id) return self.task_manager.computed_task_received(subtask_id, result, result_type) if not self.task_manager.verify_subtask(subtask_id): self._reject_subtask_result(subtask_id) return self.task_server.accept_result(subtask_id, self.result_owner) self.send(message.MessageSubtaskResultAccepted(subtask_id=subtask_id))
def dict_to_binary(cls, dictionary: dict) -> bytes: return CBORSerializer.dumps(cls.dict_to_binarizable(dictionary))
def __send_data_results(self, res): result = CBORSerializer.dumps(res.result) extra_data = {"subtask_id": res.subtask_id, "data_type": "result"} self.conn.producer = tcpnetwork.EncryptDataProducer( self.encrypt(result), self, extra_data=extra_data)
def __compress_and_dump_file(file_name, data): file_data = zlib.compress(data, 9) return CBORSerializer.dumps((os.path.basename(file_name), file_data))
def test(self): obj = MockSerializationSubject() serialized = CBORSerializer.dumps(obj) deserialized = CBORSerializer.loads(serialized) assert_properties(deserialized, obj)
def _unpack_task_result(self, trp, output_dir): tr = CBORSerializer.loads(trp) with open(os.path.join(output_dir, tr[0]), "wb") as fh: fh.write(decompress(tr[1])) return os.path.join(output_dir, tr[0])