def output(self, dataframes: [DataFrame]): """ Emits the transformed data from the filter to a log :param dataframes: The dataframes to log """ syslog_reader = DatumReader(SYSLOG_AVRO_SCHEMA, SYSLOG_AVRO_SCHEMA) for dataframe in dataframes: for row in dataframe.collect(): buffer = BytesIO() buffer.write(row.value) buffer.flush() try: buffer.seek(0) decoder = BinaryDecoder(buffer) output = syslog_reader.read(decoder) self.log(INFO, f"Received AVRO syslog -> { output }") #self.log(INFO, f"Received AVRO syslog -> { row.value }") except BaseException as e: try: buffer.seek(0) decoder = BinaryDecoder(buffer) output = syslog_reader.read(decoder) self.log(INFO, f"ERROR Received AVRO feedback -> {output}") except BaseException as e: self.log(ERROR, f"{e}")
def test_sanity(): """ Ensures that our "base" and "good" schemas are actually forwards- and backwards-compatible """ # fst schema / record fst_schema = schema.parse(open("%s/MyRecord.base.avsc" % BASE_DIR).read()) fst_writer = DatumWriter(writers_schema=fst_schema) fst_record = { "fieldWithoutDefaultValue": 0, "properField": 0, "enumField": "A", "unionField": None, "arrayField": ["world"], "mapField": {"hello": "world"}, "fixedField": "aaaaaaaaaaaaaaaa" } # sec schema / record sec_schema = schema.parse(open("%s/MyRecord.good.avsc" % BASE_DIR).read()) sec_writer = DatumWriter(writers_schema=sec_schema) sec_record = { "fieldWithoutDefaultValue": 0, "properField2": 0, "enumField": "B", "unionField": None, "arrayField": ["world"], "fixedField": "bbbbbbbbbbbbbbbb" } # Encode record w/ fst fst_buf = StringIO.StringIO() fst_encoder = BinaryEncoder(fst_buf) fst_writer.write(fst_record, fst_encoder) fst_data = fst_buf.getvalue() # Encode record w/ sec sec_buf = StringIO.StringIO() sec_encoder = BinaryEncoder(sec_buf) sec_writer.write(sec_record, sec_encoder) sec_data = sec_buf.getvalue() # writers == fst, readers == sec sec_reader = DatumReader(writers_schema=fst_schema, readers_schema=sec_schema) sec_decoder = BinaryDecoder(StringIO.StringIO(fst_data)) sec_from_fst = sec_reader.read(sec_decoder) # no exception -> good # writers == sec, readers == fst fst_reader = DatumReader(writers_schema=sec_schema, readers_schema=fst_schema) fst_decoder = BinaryDecoder(StringIO.StringIO(sec_data)) fst_from_sec = fst_reader.read(fst_decoder) # no exception -> good
def deserialize( self, data, # type: Union[bytes, BinaryIO] schema, # type: Union[str, bytes, avro.schema.Schema] ): # type: (Union[bytes, BinaryIO], Union[str, bytes, avro.schema.Schema]) -> ObjectType """Read the binary representation into a specific type. Return type will be ignored, since the schema is deduced from the provided bytes. :param data: A stream of bytes or bytes directly :type data: BinaryIO or bytes :param schema: An Avro RecordSchema :type schema: str :returns: An instantiated object :rtype: ObjectType """ if not hasattr(data, 'read'): data = BytesIO(data) reader = self.get_schema_reader(schema) with data: bin_decoder = BinaryDecoder(data) decoded_data = reader.read(bin_decoder) return decoded_data
def _decode_avro(raw_bytes): schema = SchemaFromJSONData(avro_api_schema) buffer = io.BytesIO(raw_bytes) decoder = BinaryDecoder(buffer) reader = DatumReader(schema) content = reader.read(decoder) return content
def decode( self, content, # type: Union[bytes, BinaryIO] reader, # type: DatumReader ) -> ObjectType: """Read the binary representation into a specific type. Return type will be ignored, since the schema is deduced from the provided bytes. :param content: A stream of bytes or bytes directly :type content: BinaryIO or bytes :param schema: An Avro RecordSchema :type schema: str :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. :paramtype readers_schema: str or None :returns: An instantiated object :rtype: ObjectType """ if not hasattr(content, 'read'): content = cast(bytes, content) content = BytesIO(content) with content: # type: ignore bin_decoder = BinaryDecoder(content) decoded_content = reader.read(bin_decoder) return decoded_content
def decode( self, data, # type: Union[bytes, BinaryIO] schema, # type: Union[str, bytes, avro.schema.Schema] *, readers_schema=None, # type: Optional[Union[str, bytes, avro.schema.Schema]] ) -> ObjectType: """Read the binary representation into a specific type. Return type will be ignored, since the schema is deduced from the provided bytes. :param data: A stream of bytes or bytes directly :type data: BinaryIO or bytes :param schema: An Avro RecordSchema :type schema: str :keyword readers_schema: An optional reader's schema as defined by the Apache Avro specification. :paramtype readers_schema: str or None :returns: An instantiated object :rtype: ObjectType """ if not hasattr(data, 'read'): data = BytesIO(data) reader = self.get_schema_reader(schema, readers_schema) with data: bin_decoder = BinaryDecoder(data) decoded_data = reader.read(bin_decoder) return decoded_data
def consume(self): for message in self.kfkcon: try: if self.ser_type == kfkcfg.SERIALIZATIO_JSON: message = json.loads(message.value.decode('utf-8')) elif self.ser_type == kfkcfg.SERIALIZATIO_AVRO: bytes_reader = io.BytesIO(message.value) decoder = BinaryDecoder(bytes_reader) reader = DatumReader(self.avro_schema) try: message = reader.read(decoder) print(message) except Exception as e: print(e) pass if self.print_msg: parser_logger.info( 'Message to consume: {} -- serialization: {}'.format( message, self.ser_type)) # print('Message to consume: {}'.format(message)) yield message except Exception as e: parser_logger.info( 'unable to parse the msg!: {}...error: {}'.format( message, e))
def decode(schema, msg_value): reader = DatumReader(schema) message_bytes = io.BytesIO(msg_value) message_bytes.seek(5) decoder = BinaryDecoder(message_bytes) event_dict = reader.read(decoder) return event_dict
def verify_kafka_avro_messages(record, key_capture_mode, key_capture_field, key_capture_attribute, key_schema, message_keys, message_values): for key, value in message_values.items(): assert record.get_field_data(f'/{key}').value == value # validate message key fields/attribute based on configuration if (key_capture_mode in ['RECORD_FIELD', 'RECORD_HEADER_AND_FIELD']): # the message key should have been captured into the configured field for key, value in message_keys.items(): assert record.get_field_data( f"{key_capture_field}/{key}").value == value if (key_capture_mode in ['RECORD_HEADER', 'RECORD_HEADER_AND_FIELD']): # get the base64 encoded Avro message key encoded = record.header['values'][key_capture_attribute] # decode into bytes key_bytes = base64.standard_b64decode(encoded) # create an Avro binary decoder based on those bytes decoder = BinaryDecoder(io.BytesIO(key_bytes)) # parse the key schema out of the record header decoded_key_schema = confluent_kafka.avro.loads( record.header['values']['avroKeySchema']) # ensure the parsed key schema matches the one we actually produced, earlier assert decoded_key_schema == key_schema # create a DatumReader to read a full Avro record (the key) reader = DatumReader(decoded_key_schema) decoded_avro_key = reader.read(decoder) # assert the values from the Avro record match what's expected for key, value in message_keys.items(): assert decoded_avro_key[key] == value
def respond(self, call_request): buffer_reader = io.BytesIO(call_request) buffer_decoder = BinaryDecoder(buffer_reader) buffer_writer = io.BytesIO() buffer_encoder = BinaryEncoder(buffer_writer) error = None response_metadata = {} try: remote_protocol = self.process_handshake(buffer_decoder, buffer_encoder) if remote_protocol is None or self.local_protocol is None: return buffer_writer.getvalue() DatumReader(schema.parse( '{"type": "map", "values": "bytes"}')).read(buffer_decoder) remote_message_name = buffer_decoder.read_utf8() remote_message = remote_protocol.messages.get(remote_message_name) if remote_message is None: fail_msg = 'Unknown remote message: %s' % remote_message_name raise schema.AvroException(fail_msg) local_message = self.local_protocol.messages.get( remote_message_name) if local_message is None: fail_msg = 'Unknown local message: %s' % remote_message_name raise schema.AvroException(fail_msg) writers_schema = remote_message.request readers_schema = local_message.request request = self.read_request(writers_schema, readers_schema, buffer_decoder) response = None try: response = self.invoke(self.local_protocol, local_message, request) except AvroRemoteException as e: error = e except Exception as e: error = AvroRemoteException(str(e)) DatumWriter( schema.parse('{"type": "map", "values": "bytes"}')).write( response_metadata, buffer_encoder) buffer_encoder.write_boolean(error is not None) if error is None: writers_schema = local_message.response self.write_response(writers_schema, response, buffer_encoder) else: writers_schema = local_message.errors self.write_error(writers_schema, error, buffer_encoder) except schema.AvroException as e: error = AvroRemoteException(str(e)) buffer_encoder = BinaryEncoder(io.BytesIO()) DatumWriter( schema.parse('{"type": "map", "values": "bytes"}')).write( response_metadata, buffer_encoder) buffer_encoder.write_boolean(True) self.write_error(schema.parse('["string"]'), error, buffer_encoder) return buffer_encoder.writer.getvalue() return buffer_writer.getvalue()
def avro_deserialization(value_schema,x): datum_reader = DatumReader(writer_schema=value_schema, reader_schema=value_schema) bytes_io = BytesIO(x) decoder = BinaryDecoder(bytes_io) deserialized_x = datum_reader.read(decoder) bytes_io.close() return deserialized_x
def avro_decode(cls, binary_data, schema=None): """avro 反序列化二进制数据为json :param binary_data: :param schema: :return: """ bio = BytesIO(binary_data) binary_decoder = BinaryDecoder(bio) return DatumReader(schema or cls.REQUEST_SCHEMA).read(binary_decoder)
def binToObj(ab): bytes_reader = io.BytesIO(ab) decoder = BinaryDecoder(bytes_reader) reader = DatumReader(sc) while True: try: rec = reader.read(decoder) print(rec) except: break
def deserialize(flight_info_bytes): if flight_info_bytes is not None: bytes_reader = BytesIO(flight_info_bytes) decoder = BinaryDecoder(bytes_reader) schema_flight_info = Parse( open(dir_path + "/flight-info.schema.avsc", "rb").read()) reader = DatumReader(schema_flight_info) flight_info = reader.read(decoder) return [{"id": "1"}, {"id": "2"}] else: return None
def unpack(self, payload): magic, schema_id = struct.unpack('>bi', payload[:5]) if magic == MAGIC_BYTES: schema = self.RegistryClient.get_by_id(schema_id) reader = DatumReader(schema) output = BinaryDecoder(io.BytesIO(payload[5:])) abc = reader.read(output) return abc else: return payload.decode()
def deserialize_avro(binary_data, schema): """ Function used to deserialize an avro binary data :param schema: avro schema of binary data :param binary_data: event data in binary encoded (bytes) :return: deserialized data and corresponding schema """ bytes_reader = io.BytesIO(binary_data) decoder = BinaryDecoder(bytes_reader) reader = DatumReader(schema) data = reader.read(decoder) return data, schema
def read_value(schema: TypedSchema, bio: io.BytesIO): if schema.schema_type is SchemaType.AVRO: reader = DatumReader(schema.schema) return reader.read(BinaryDecoder(bio)) if schema.schema_type is SchemaType.JSONSCHEMA: value = load(bio) try: schema.schema.validate(value) except ValidationError as e: raise InvalidPayload from e return value raise ValueError("Unknown schema type")
def deserialize(flight_info_bytes): if flight_info_bytes is not None: bytes_reader = BytesIO(flight_info_bytes) decoder = BinaryDecoder(bytes_reader) schema_flight_info = Parse( open(dir_path + "/flight-info.schema.avsc", "rb").read()) reader = DatumReader(schema_flight_info) flight_info = reader.read(decoder) return json.dumps([{"id": 907955534287978496}]) else: return None
def decode_avro(self, payload): magic, schema_id = struct.unpack('>bi', payload[:5]) # Get Schema registry # Avro value format if magic == MAGIC_BYTES: schema = self.register_client.get_by_id(schema_id) reader = DatumReader(schema) output = BinaryDecoder(io.BytesIO(payload[5:])) decoded = reader.read(output) return decoded, schema.name # no magic bytes, something is wrong else: raise ValueError
def parse_avro_msg(msg, avro_schema): """ Parses an avro record using a specified avro schema Args: :msg: the avro message to parse :avro_schema: the avro schema Returns: The parsed/decoded message """ reader = DatumReader(avro_schema) message_bytes = BytesIO(msg) decoder = BinaryDecoder(message_bytes) return reader.read(decoder)
def _decode_avro(self, avro_data): """ Decode Avro Message. Args: avro_data (binary): Avro data to decode. Returns: dict """ bytes_reader = io.BytesIO(avro_data.value) decoder = BinaryDecoder(bytes_reader) reader = DatumReader(self.schema) return reader.read(decoder)
def unpack(self, payload): MAGIC_BYTES = 0 magic, schema_id = struct.unpack('>bi', payload[:5]) # Get Schema registry # Avro value format if magic == MAGIC_BYTES: schema = self.register_client.get_by_id(schema_id) reader = DatumReader(schema) output = BinaryDecoder(io.BytesIO(payload[5:])) abc = reader.read(output) return abc # String key else: # Timestamp is inside my key return payload[:-8].decode()
def decode_avro(msg_value, reader): """Function to decode the bytes from the Avro serilization Args: msg_value ([bytes]): message to deserialize reader ([DatumReader]): special variable of Avro schema for the deserilization Returns: [dict]: deserialize data dictionary """ message_bytes = io.BytesIO(msg_value) decoder = BinaryDecoder(message_bytes) event_dict = reader.read(decoder) return event_dict
def _unpack(self, payload): magic, schema_id = struct.unpack('>bi', payload[:5]) # Get Schema registry # Avro value format if magic == self.MAGIC_BYTES: schema = self.register_client.get_by_id(schema_id) reader = DatumReader(schema) output = BinaryDecoder(io.BytesIO(payload[5:])) abc = reader.read(output) return abc # String key else: # If KSQL payload, exclude timestamp which is inside the key. # payload[:-8].decode() return payload.decode()
def callback(message): # Get the message serialization type. encoding = message.attributes.get("googclient_schemaencoding") # Deserialize the message data accordingly. if encoding == "BINARY": bout = io.BytesIO(message.data) decoder = BinaryDecoder(bout) reader = DatumReader(avro_schema) message_data = reader.read(decoder) print(f"Received a binary-encoded message:\n{message_data}") elif encoding == "JSON": message_data = json.loads(message.data) print(f"Received a JSON-encoded message:\n{message_data}") else: print(f"Received a message with no encoding:\n{message}") message.ack()
def unpack(payload, windowed, register_client): magic, schema_id = struct.unpack('>bi', payload[:5]) # Get Schema registry # Avro value format if magic == MAGIC_BYTES: schema = register_client.get_by_id(schema_id) reader = DatumReader(schema) output = BinaryDecoder(io.BytesIO(payload[5:])) abc = reader.read(output) return abc # String key else: # If Windowed KSQL payload if windowed == 'TRUE': return payload[:-8].decode() elif windowed == 'FALSE': return payload.decode()
def extract_messages(ams, ingest_sub, bulk_size, schema, verify): # consume metric data messages consumed_msgs = ams.pull_sub(ingest_sub, num=bulk_size, return_immediately=True, verify=verify) # initialise the avro reader avro_reader = DatumReader(writers_schema=schema) # all the decoded messages that will be returned decoded_msgs = [] # decode the messages for msg in consumed_msgs: try: # decode the data field again using the provided avro schema msg_bytes = BytesIO(msg[1].get_data()) msg_decoder = BinaryDecoder(msg_bytes) avro_msg = avro_reader.read(msg_decoder) # check that the tags field is present if avro_msg["tags"] is None: raise KeyError("tags field is empty") # append to decoded messages decoded_msgs.append((msg[0], avro_msg)) except Exception as e: LOGGER.warning( "Could not extract data from ams message {}, {}".format( msg[0], e.message)) last_msg_id = "-1" if len(consumed_msgs) > 0: last_msg_id = consumed_msgs.pop()[0] return decoded_msgs, last_msg_id
def decode_avro(data): decoder = BinaryDecoder(BytesIO(data)) new_data = datum_reader.read(decoder) return new_data
def with_deserialization(self, *args, **kwargs): ret = meth(self, *args, **kwargs) f = StringIO(ret) dec = BinaryDecoder(f) return datum_reader.read(dec)
def decoder(binary_data): bio = BytesIO(binary_data) binary_decoder = BinaryDecoder(bio) # return "yangkun" schema = avro.schema.parse(open("user.avsc", "rb").read()) return DatumReader(schema).read(binary_decoder)