def from_zmsg(self, zmsg: TZMQMessage) -> None:
"""Populate message data from sequence of ZMQ data frames.
Arguments:
zmsg: Sequence of frames that should be deserialized.
Raises:
InvalidMessageError: If message is not a valid protocol message.
"""
try:
self.msg_type = MsgType(unpack('!H', zmsg[0])[0])
if self.msg_type is MsgType.READY:
proto = protobuf.create_message(PROTO_PEER, zmsg[1])
self.peer = PeerDescriptor.from_proto(proto)
proto = protobuf.create_message(protobuf.PROTO_STRUCT, zmsg[2])
self.endpoints = {}
for k, v in protobuf.struct2dict(proto).items():
for i in range(len(v)):
v[i] = ZMQAddress(v[i])
self.endpoints[k] = v
elif self.msg_type is MsgType.ERROR:
self.error = zmsg[1].decode('utf8')
elif self.msg_type is MsgType.FINISHED:
self.outcome = Outcome(zmsg[1].decode())
self.details = [
v.decode('utf8', errors='replace') for v in zmsg[2:]
]
elif self.msg_type is MsgType.REQUEST:
self.request = Request(zmsg[1])
if self.request is Request.CONFIGURE:
self.config = protobuf.create_message(
PROTO_CONFIG, zmsg[2])
except Exception as exc:
raise InvalidMessageError("Invalid message") from exc
def from_zmsg(self, zmsg: TZMQMessage) -> None:
"""Populate message data from sequence of ZMQ data frames.
Arguments:
zmsg: Sequence of frames that should be deserialized.
Raises:
InvalidMessageError: If message is not a valid protocol message.
"""
try:
control_byte, flags, self.type_data = unpack(
HEADER_FMT, zmsg.pop(0))
self.msg_type = MsgType(control_byte >> 3)
self.flags = MsgFlag(flags)
if self.msg_type is MsgType.OPEN:
self.data_frame = create_message(PROTO_OPEN)
self.data_frame.ParseFromString(zmsg.pop(0))
elif self.msg_type is MsgType.DATA:
self.data_frame = zmsg.pop(0) if zmsg else None
elif self.msg_type is MsgType.CLOSE:
self.type_data = ErrorCode(self.type_data)
self.data_frame = []
while zmsg:
err = create_message(PROTO_ERROR)
err.ParseFromString(zmsg.pop(0))
self.data_frame.append(err)
except Exception as exc:
raise InvalidMessageError("Invalid message") from exc
def handle_input_accept_client(self, channel: Channel,
session: FBDPSession) -> None:
"""Event handler executed when client connects to INPUT data pipe via OPEN message.
Arguments:
channel: Channel associated with data pipe.
session: Session associated with client.
The session attributes `data_pipe`, `pipe_socket`, `data_format` and `params`
contain information sent by client, and the event handler validates the request.
If request should be rejected, it raises the `StopError` exception with `code`
attribute containing the `ErrorCode` to be returned in CLOSE message.
"""
super().handle_input_accept_client(channel, session)
fmt = cast(MIME, session.data_format)
if fmt.mime_type != MIME_TYPE_PROTO:
raise StopError(
f"MIME type '{fmt.mime_type}' is not a valid input format",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
proto_class = fmt.params.get('type')
if self.data:
if self.data.DESCRIPTOR.full_name != proto_class:
raise StopError(
f"Protobuf message type '{proto_class}' not allowed",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
else:
if not is_msg_registered(proto_class):
raise StopError(
f"Unknown protobuf message type '{proto_class}'",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
self.data = create_message(proto_class)
def initialize(self, config: ProtoFilterConfig) -> None:
"""Verify configuration and assemble component structural parts.
"""
super().initialize(config)
self.log_context = 'main'
#
self.include_func: Callable = None
self.exclude_func: Callable = None
self.data: Any = None
self.fmt: MIME = None
#
if config.include_expr.value is not None:
self.include_func = config.include_expr.value.get_callable('data',
{'datetime': datetime,})
if config.include_func.value is not None:
self.include_func = config.include_func.value
if config.exclude_expr.value is not None:
self.exclude_func = config.exclude_expr.value.get_callable('data',
{'datetime': datetime,})
if config.exclude_func.value is not None:
self.exclude_func = config.exclude_func.value
#
self.fmt = config.input_pipe_format.value
proto_class = self.fmt.params.get('type')
if not is_msg_registered(proto_class):
raise StopError(f"Unknown protobuf message type '{proto_class}'",
code = ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
self.data = create_message(proto_class)
def test_messages(self):
register_decriptor(DESCRIPTOR)
#
msg = create_message(STATE_MSG_TYPE_NAME)
self.assertIsNotNone(msg)
self.assertEqual(get_enum_field_type(msg, 'test'), ENUM_TYPE_NAME)
#
msg.name = 'State.NAME'
msg.test = 1
# Errors
with self.assertRaises(KeyError) as cm:
create_message('NOT_REGISTERED')
self.assertEqual(cm.exception.args, ("Unregistered protobuf message 'NOT_REGISTERED'",))
with self.assertRaises(KeyError) as cm:
get_enum_field_type(msg, 'BAD_FIELD')
self.assertEqual(cm.exception.args, ("Message does not have field 'BAD_FIELD'",))
def initialize(self, config: ProtoPrinterConfig) -> None:
"""Verify configuration and assemble component structural parts.
"""
super().initialize(config)
self.log_context = 'main'
#
self.transform_func: Callable[[Any], str] = None
self.fmt: str = None
self.data: Any = None
self.charset = 'ascii'
self.errors = 'strict'
self.utils = TransformationUtilities()
#
if config.input_pipe_format.value is not None:
self.data = create_message(
config.input_pipe_format.value.params.get('type'))
#
if config.template.value is not None:
self.transform_func = self.__format_data
self.fmt = 'f"""' + config.template.value + '"""'
else:
self.transform_func = config.func.value
#
if self.output_pipe_mode is SocketMode.CONNECT:
self.output_protocol.on_init_session = self.handle_init_session
def initialize(self, config: FbTraceParserConfig) -> None:
"""Verify configuration and assemble component structural parts.
"""
super().initialize(config)
self.log_context = 'main'
#
self.proto = create_message(TRACE_PROTO)
self.entry_buf: List[str] = []
self.parser: TraceParser = TraceParser()
self.input_lefover = None
#
self.data_map: Dict = {
AttachmentInfo: self.store_att_info,
TransactionInfo: self.store_tra_info,
ServiceInfo: self.store_svc_info,
SQLInfo: self.store_sql_info,
ParamSet: self.store_param_set,
EventTraceInit: self.store_trace_init,
EventTraceSuspend: self.store_trace_suspend,
EventTraceFinish: self.store_trace_finish,
EventCreate: self.store_db_create,
EventDrop: self.store_db_drop,
EventAttach: self.store_db_attach,
EventDetach: self.store_db_detach,
EventTransactionStart: self.store_tra_start,
EventCommit: self.store_commit,
EventRollback: self.store_rollback,
EventCommitRetaining: self.store_commit_retain,
EventRollbackRetaining: self.store_rollback_retain,
EventPrepareStatement: self.store_stm_prepare,
EventStatementStart: self.store_stm_start,
EventStatementFinish: self.store_smt_finish,
EventFreeStatement: self.store_stm_free,
EventCloseCursor: self.store_cursor_close,
EventTriggerStart: self.store_trigger_start,
EventTriggerFinish: self.store_trigger_finish,
EventProcedureStart: self.store_proc_start,
EventProcedureFinish: self.store_proc_finish,
EventServiceAttach: self.store_svc_attach,
EventServiceDetach: self.store_svc_detach,
EventServiceStart: self.store_svc_start,
EventServiceQuery: self.store_svc_query,
EventSetContext: self.store_ctx_set,
EventError: self.store_error,
EventWarning: self.store_warning,
EventServiceError: self.store_svc_error,
EventServiceWarning: self.store_svc_warning,
EventSweepStart: self.store_swp_start,
EventSweepProgress: self.store_swp_progress,
EventSweepFinish: self.store_swp_finish,
EventSweepFailed: self.store_swp_fail,
EventBLRCompile: self.store_blr_compile,
EventBLRExecute: self.store_blr_exec,
EventDYNExecute: self.store_dyn_exec,
EventUnknown: self.store_unknown
}
#
if self.input_pipe_mode is SocketMode.CONNECT:
self.input_protocol.on_init_session = self.handle_init_session
def validate(self, zmsg: TZMQMessage) -> None:
"""Verifies that sequence of ZMQ data frames is a valid protocol message.
If this validation passes without exception, then `.parse()` of the same message
must be successful as well.
Arguments:
zmsg: ZeroMQ multipart message.
Raises:
InvalidMessageError: If ZMQ message is not a valid protocol message.
"""
if not zmsg:
raise InvalidMessageError("Empty message")
try:
msg_type = MsgType(unpack('!H', zmsg[0]))
except Exception as exc:
raise InvalidMessageError("Invalid message type") from exc
if msg_type is MsgType.READY:
try:
PeerDescriptor.from_proto(
protobuf.create_message(PROTO_PEER, zmsg[1]))
except Exception as exc:
raise InvalidMessageError(
"Invalid data: peer descriptor") from exc
try:
protobuf.create_message(protobuf.PROTO_STRUCT, zmsg[2])
except Exception as exc:
raise InvalidMessageError("Invalid data: endpoints") from exc
elif msg_type is MsgType.ERROR:
try:
zmsg[1].decode('utf8')
except Exception as exc:
raise InvalidMessageError(
"Invalid data: error message") from exc
elif msg_type is MsgType.REQUEST:
try:
req = Request(zmsg[1])
except Exception as exc:
raise InvalidMessageError("Invalid request code") from exc
if req is Request.CONFIGURE:
try:
protobuf.create_message(PROTO_CONFIG, zmsg[2])
except Exception as exc:
raise InvalidMessageError("Invalid data: config") from exc
def note_exception(self, exc: Exception):
"""Store information from exception into CLOSE Message.
"""
assert self.msg_type is MsgType.CLOSE
errdesc = create_message(PROTO_ERROR)
if hasattr(exc, 'code'):
errdesc.code = getattr(exc, 'code')
errdesc.description = str(exc)
self.data_frame.append(errdesc)
def request_config_msg(self, config: Config = None) -> ICCPMessage:
"""Returns REQUEST/CONFIG control message.
"""
msg: ICCPMessage = self.message_factory()
msg.msg_type = MsgType.REQUEST
msg.request = Request.CONFIGURE
msg.config = protobuf.create_message(PROTO_CONFIG)
if config:
config.save_proto(msg.config)
return msg
def copy(self) -> Message:
"""Returns copy of the message.
"""
msg: FBDPMessage = self.__class__()
msg.msg_type = self.msg_type
msg.flags = self.flags
msg.type_data = self.type_data
if self.msg_type is MsgType.OPEN:
msg.data_frame = create_message(PROTO_OPEN)
msg.data_frame.CopyFrom(self.data_frame)
elif self.msg_type is MsgType.CLOSE:
msg.data_frame = []
for frame in self.data_frame:
err = create_message(PROTO_ERROR)
err.CopyFrom(frame)
msg.data_frame.append(err)
else:
msg.data_frame = self.data_frame
return msg
def as_proto(self) -> Any:
"""Returns `firebird.butler.PeerIdentification` protobuf message initialized
from instance data.
"""
msg = create_message(PROTO_PEER)
msg.uid = self.uid.bytes
msg.pid = self.pid
msg.host = self.host
if self.supplement is not None:
sup = msg.supplement.add()
sup.Pack(dict2struct(self.supplement))
return msg
def initialize(self, config: FbLogParserConfig) -> None:
"""Verify configuration and assemble component structural parts.
"""
super().initialize(config)
self.log_context = 'main'
#
self.proto = create_message(LOG_PROTO)
self.entry_buf: List[str] = []
self.parser: LogParser = LogParser()
self.input_lefover = None
#
if self.input_pipe_mode is SocketMode.CONNECT:
self.input_protocol.on_init_session = self.handle_init_session
def handle_init_session(self, channel: Channel,
session: FBDPSession) -> None:
"""Event executed from `send_open()` to set additional information to newly
created session instance.
"""
# cache attributes
if cast(MIME, session.data_format).mime_type == MIME_TYPE_TEXT:
session.charset = cast(MIME, session.data_format).params.get(
'charset', 'ascii')
session.errors = cast(MIME, session.data_format).params.get(
'errors', 'strict')
elif cast(MIME, session.data_format).mime_type == MIME_TYPE_PROTO:
session.proto = create_message(
cast(MIME, session.data_format).params.get('type'))
def from_proto(cls, proto: Any) -> PeerDescriptor:
"""Creates new PeerDescriptor from `firebird.butler.PeerIdentification` protobuf
message.
"""
if proto.DESCRIPTOR.full_name != PROTO_PEER:
raise ValueError("PeerIdentification protobuf message required")
data = None
if proto.supplement:
for i in proto.supplement:
if i.TypeName() == PROTO_STRUCT:
s = create_message(PROTO_STRUCT)
i.Unpack(s)
data = struct2dict(s)
break
return cls(uuid.UUID(bytes=proto.uid), proto.pid, proto.host, data)
def copy(self) -> Message:
"""Returns copy of the message.
"""
msg = self.__class__()
msg.msg_type = self.msg_type
if self.msg_type is MsgType.READY:
msg.peer = self.peer.copy()
msg.endpoints = self.endpoints.copy()
elif self.msg_type is MsgType.ERROR:
msg.error = self.error
elif self.msg_type is MsgType.REQUEST:
msg.request = self.request
if self.request is Request.CONFIGURE:
msg.config = protobuf.create_message(PROTO_CONFIG)
msg.config.CopyFrom(self.config)
return msg
def handle_accept_client(self, channel: Channel,
session: FBDPSession) -> None:
"""Event handler executed when client connects to the data pipe via OPEN message.
Arguments:
channel: Channel associated with data pipe.
session: Session associated with client.
The session attributes `data_pipe`, `pipe_socket`, `data_format` and `params`
contain information sent by client, and the event handler validates the request.
If request should be rejected, it raises the `StopError` exception with `code`
attribute containing the `ErrorCode` to be returned in CLOSE message.
"""
super().handle_accept_client(channel, session)
if cast(MIME, session.data_format).mime_type not in SUPPORTED_MIME:
raise StopError(
f"MIME type '{cast(MIME, session.data_format).mime_type}' is not a valid input format",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
if cast(MIME, session.data_format).mime_type == MIME_TYPE_TEXT:
for param in cast(MIME, session.data_format).params.keys():
if param not in ('charset', 'errors'):
raise StopError(f"Unknown MIME parameter '{param}'",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
# cache attributes
session.charset = cast(MIME, session.data_format).params.get(
'charset', 'ascii')
session.errors = cast(MIME, session.data_format).params.get(
'errors', 'strict')
elif cast(MIME, session.data_format).mime_type == MIME_TYPE_PROTO:
for param in cast(MIME, session.data_format).params.keys():
if param != 'type':
raise StopError(f"Unknown MIME parameter '{param}'",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
proto_class = cast(MIME, session.data_format).params.get('type')
if not is_msg_registered(proto_class):
raise StopError(
f"Unknown protobuf message type '{proto_class}'",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
# cache attributes
session.proto = create_message(proto_class)
# Client reqest is ok, we'll open the file we are configured to work with.
self._open_file()
def initialize(self, config: ProtoAggregatorConfig) -> None:
"""Verify configuration and assemble component structural parts.
"""
super().initialize(config)
self.log_context = 'main'
#
self.data: Any = None
self.group_by: List[GroupByItem] = []
self.agg_defs: List[str] = []
self.aggregates: Dict[Tuple, AggregateItem] = {}
#
for item in config.group_by.value:
self.group_by.append(GroupByItem(item))
for item in config.aggregate.value:
self.agg_defs.append(item)
#
proto_class = config.input_pipe_format.value.params.get('type')
if not is_msg_registered(proto_class):
raise StopError(f"Unknown protobuf message type '{proto_class}'",
code=ErrorCode.DATA_FORMAT_NOT_SUPPORTED)
self.data = create_message(proto_class)
def create_message_for(self,
msg_type: MsgType,
type_data: int = 0,
flags: MsgFlag = None) -> FBDPMessage:
"""Returns message of particular FBDP message type.
Arguments:
msg_type: Type of message to be created.
type_data: Message control data.
flags: Message flags.
"""
msg: FBDPMessage = self.message_factory()
msg.msg_type = msg_type
msg.type_data = type_data
if flags is not None:
msg.flags = flags
if msg.msg_type is MsgType.OPEN:
msg.data_frame = create_message(PROTO_OPEN)
elif msg.msg_type is MsgType.CLOSE:
msg.data_frame = []
return msg
def finish_input_processing(self, channel: Channel, session: FBDPSession,
code: ErrorCode) -> None:
"""Called when input pipe is closed while output pipe will remain open.
When code is ErrorCode.OK, the input was closed normally. Otherwise it indicates
the type of problem that caused the input to be closed.
Arguments:
channel: Channel associated with data pipe.
session: Session associated with peer.
code: Input pipe closing ErrorCode.
"""
output_data = create_message(AGGREGATE_PROTO)
batch = []
for key, items in self.aggregates.items():
output_data.Clear()
i = 0
for grp in self.group_by:
output_data.data[grp.name] = key[i]
i += 1
for item in items:
output_data.data[item.name] = item.get_result()
batch.append(output_data.SerializeToString())
self.store_batch_output(batch)
def validate(self, zmsg: TZMQMessage) -> None:
"""Verifies that sequence of ZMQ data frames is a valid protocol message.
If this validation passes without exception, then `.parse()` of the same message
must be successful as well.
Arguments:
zmsg: ZeroMQ multipart message.
Raises:
InvalidMessageError: If ZMQ message is not a valid protocol message.
"""
if not zmsg:
raise InvalidMessageError("Empty message")
fbdp_header = zmsg[0]
if len(fbdp_header) != 8:
raise InvalidMessageError("Message header must be 8 bytes long")
try:
fourcc, control_byte, flags, _ = unpack(HEADER_FMT_FULL,
fbdp_header)
except Exception as exp:
raise InvalidMessageError("Invalid control frame") from exp
if fourcc != FOURCC:
raise InvalidMessageError("Invalid FourCC")
if (control_byte & VERSION_MASK) != self.REVISION:
raise InvalidMessageError("Invalid protocol version")
if (flags | 3) > 3:
raise InvalidMessageError("Invalid flags")
try:
message_type = MsgType(control_byte >> 3)
except ValueError:
raise InvalidMessageError(
f"Illegal message type {control_byte >> 3}")
if message_type is MsgType.OPEN:
if len(zmsg) != 2:
raise InvalidMessageError("OPEN message must have a dataframe")
try:
fpb = create_message(PROTO_OPEN)
fpb.ParseFromString(zmsg[1])
if not fpb.data_pipe:
raise ValueError("Missing 'data_pipe' specification")
pipe_socket = PipeSocket(fpb.pipe_socket)
if pipe_socket is PipeSocket.UNKNOWN_PIPE_SOCKET:
raise ValueError("Invalid 'pipe_socket'")
if not fpb.data_format:
raise ValueError("Missing 'data_format' specification")
except Exception as exc:
raise InvalidMessageError(
"Invalid data frame for OPEN message") from exc
elif (message_type is MsgType.CLOSE and len(zmsg) > 1):
fpb = create_message(PROTO_ERROR)
for frame in zmsg[1:]:
fpb.ParseFromString(frame)
if not fpb.description:
raise InvalidMessageError("Missing error description")
elif (message_type is MsgType.DATA and len(zmsg) > 2):
raise InvalidMessageError(
"DATA message may have only one data frame")
elif (message_type in (MsgType.READY, MsgType.NOOP) and len(zmsg) > 1):
raise InvalidMessageError(
"Data frames not allowed for READY and NOOP messages")
