def __init__(self, from_packet, event_size, table_map, ctl_connection):
super(TableMapEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection)
# Post-Header
self.table_id = self._read_table_id()
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self.__get_table_informations(
self.schema, self.table)
#Read columns meta data
column_types = list(self.packet.read(self.column_count))
metadata_length = self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
column_schema = self.column_schemas[i]
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
def __init__(self, from_packet, event_size, table_map, ctl_connection):
super(TableMapEvent, self).__init__(from_packet, event_size,
table_map, ctl_connection)
# Post-Header
self.table_id = self._read_table_id()
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self._ctl_connection._get_table_information(self.schema, self.table)
# Read columns meta data
column_types = list(self.packet.read(self.column_count))
self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
column_schema = self.column_schemas[i]
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
self.table_obj = Table(self.column_schemas, self.table_id, self.schema,
self.table, self.columns)
def _read_packet(self, packet_type=MysqlPacket):
"""Read an entire "mysql packet" in its entirety from the network
and return a MysqlPacket type that represents the results.
"""
buff = b''
try:
while True:
packet_header = yield from self._reader.readexactly(4)
# logger.debug(_convert_to_str(packet_header))
packet_length_bin = packet_header[:3]
# TODO: check sequence id
# packet_number
byte2int(packet_header[3])
# pad little-endian number
bin_length = packet_length_bin + b'\0'
bytes_to_read = struct.unpack('<I', bin_length)[0]
recv_data = yield from self._reader.readexactly(bytes_to_read)
# logger.debug(dump_packet(recv_data))
buff += recv_data
if bytes_to_read < MAX_PACKET_LEN:
break
except (OSError, EOFError) as exc:
msg = "MySQL server has gone away (%s)"
raise OperationalError(2006, msg % (exc, )) from exc
packet = packet_type(buff, self._encoding)
packet.check_error()
return packet
def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
super(TableMapEvent, self).__init__(from_packet, event_size,
table_map, ctl_connection, **kwargs)
self.__only_tables = kwargs["only_tables"]
self.__only_schemas = kwargs["only_schemas"]
self.__freeze_schema = kwargs["freeze_schema"]
# Post-Header
self.table_id = self._read_table_id()
if self.table_id in table_map and self.__freeze_schema:
self._processed = False
return
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
if self.__only_tables is not None and self.table not in self.__only_tables:
self._processed = False
return
if self.__only_schemas is not None and self.schema not in self.__only_schemas:
self._processed = False
return
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self._ctl_connection._get_table_information(self.schema, self.table)
# Read columns meta data
column_types = list(self.packet.read(self.column_count))
self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
column_schema = self.column_schemas[i]
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
self.table_obj = Table(self.column_schemas, self.table_id, self.schema,
self.table, self.columns)
self.__hashes = {}
def __init__(self, from_packet, event_size, table_map, ctl_connection,
**kwargs):
super(GtidEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection, **kwargs)
self.commit_flag = byte2int(self.packet.read(1)) == 1
self.sid = self.packet.read(16)
self.gno = struct.unpack('<Q', self.packet.read(8))[0]
self.lt_type = byte2int(self.packet.read(1))
if self.mysql_version >= (5, 7):
self.last_committed = struct.unpack('<Q', self.packet.read(8))[0]
self.sequence_number = struct.unpack('<Q', self.packet.read(8))[0]
def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
super(TableMapEvent, self).__init__(from_packet, event_size,
table_map, ctl_connection, **kwargs)
self.__only_tables = kwargs["only_tables"]
self.__only_schemas = kwargs["only_schemas"]
self.__freeze_schema = kwargs["freeze_schema"]
# Post-Header
self.table_id = self._read_table_id()
if self.table_id in table_map and self.__freeze_schema:
self._processed = False
return
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
if self.__only_tables is not None and self.table not in self.__only_tables:
self._processed = False
return
if self.__only_schemas is not None and self.schema not in self.__only_schemas:
self._processed = False
return
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self._ctl_connection._get_table_information(self.schema, self.table)
# Read columns meta data
column_types = list(self.packet.read(self.column_count))
self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
column_schema = self.column_schemas[i]
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
self.table_obj = Table(self.column_schemas, self.table_id, self.schema,
self.table, self.columns)
def __init__(self, binlog, use_checksum):
self.read_bytes = 0 # -1 because we ignore the ok byte
self.__data_buffer = b'' # Used when we want to override a value in the data buffer
# Binlog event header: https://dev.mysql.com/doc/internals/en/event-header-fields.html
event_header = binlog.read(19)
unpack = struct.unpack('<IcIIIH', event_header)
self.timestamp = unpack[0]
self.event_type = byte2int(unpack[1])
self.server_id = unpack[2]
self.event_size = unpack[3]
self.log_pos = unpack[4] # Position of the next event
self.flags = unpack[5]
# MySQL 5.6 and more if binlog-checksum = CRC32
event_body_size = self.event_size - 23 if use_checksum else self.event_size - 19
self._event_body = binlog.read(event_body_size)
self._event_body_pos = 0
event_class = self._event_map.get(self.event_type,
event.NotImplementedEvent)
self.event = event_class(self,
event_body_size, {},
None,
only_tables=[],
ignored_tables=[],
only_schemas=[],
ignored_schemas=[],
freeze_schema=[])
def __init__(self, from_packet, event_size, table_map, ctl_connection,
**kwargs):
super(QueryEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection, **kwargs)
# Post-header
self.slave_proxy_id = self.packet.read_uint32()
self.execution_time = self.packet.read_uint32()
self.schema_length = byte2int(self.packet.read(1))
self.error_code = self.packet.read_uint16()
self.status_vars_length = self.packet.read_uint16()
# Payload
self.status_vars = self.packet.read(self.status_vars_length)
self.schema = self.packet.read(self.schema_length)
self.packet.advance(1)
self.query = self.packet.read(event_size - 13 -
self.status_vars_length -
self.schema_length - 1)
try:
self.query = self.query.decode("utf-8")
except UnicodeDecodeError:
self.query = self.query.decode("latin-1")
def __init__(self, from_packet, table_map, ctl_connection, use_checksum):
if not from_packet.is_ok_packet():
raise ValueError("Cannot create %s object from invalid packet type" % self.__class__.__name__)
# -1 because we ignore the ok byte
self.read_bytes = 0
# Used when we want to override a value in the data buffer
self.__data_buffer = b""
# Ok Value
self.packet = from_packet
self.packet.advance(1)
self.charset = ctl_connection.charset
# Header
self.timestamp = struct.unpack("<I", self.packet.read(4))[0]
self.event_type = byte2int(self.packet.read(1))
self.server_id = struct.unpack("<I", self.packet.read(4))[0]
self.event_size = struct.unpack("<I", self.packet.read(4))[0]
# position of the next event
self.log_pos = struct.unpack("<I", self.packet.read(4))[0]
self.flags = struct.unpack("<H", self.packet.read(2))[0]
# MySQL 5.6 and more if binlog-checksum = CRC32
if use_checksum:
event_size_without_header = self.event_size - 23
else:
event_size_without_header = self.event_size - 19
event_class = self.__event_map.get(self.event_type, event.NotImplementedEvent)
self.event = event_class(self, event_size_without_header, table_map, ctl_connection)
def __init__(self, from_packet, table_map, ctl_connection, use_checksum):
if not from_packet.is_ok_packet():
raise ValueError(
"Cannot create %s object from invalid packet type" %
self.__class__.__name__)
# -1 because we ignore the ok byte
self.read_bytes = 0
# Used when we want to override a value in the data buffer
self.__data_buffer = b''
# Ok Value
self.packet = from_packet
self.packet.advance(1)
self.charset = ctl_connection.charset
# Header
self.timestamp = struct.unpack('<I', self.packet.read(4))[0]
self.event_type = byte2int(self.packet.read(1))
self.server_id = struct.unpack('<I', self.packet.read(4))[0]
self.event_size = struct.unpack('<I', self.packet.read(4))[0]
# position of the next event
self.log_pos = struct.unpack('<I', self.packet.read(4))[0]
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# MySQL 5.6 and more if binlog-checksum = CRC32
if use_checksum:
event_size_without_header = self.event_size - 23
else:
event_size_without_header = self.event_size - 19
event_class = self.__event_map.get(self.event_type,
event.NotImplementedEvent)
self.event = event_class(self, event_size_without_header, table_map,
ctl_connection)
def __init__(self, from_packet, table_map, ctl_connection, use_checksum,
allowed_events, only_tables, ignored_tables, only_schemas,
ignored_schemas, freeze_schema,
fail_on_table_metadata_unavailable,
file_name_binlog): #DNX-GET-LINE
# -1 because we ignore the ok byte
self.read_bytes = 0
# Used when we want to override a value in the data buffer
self.__data_buffer = b''
self.packet = from_packet
self.charset = ctl_connection.charset
# OK value
# timestamp
# event_type
# server_id
# log_pos
# flags
unpack = struct.unpack('<cIcIIIH', self.packet.read(20))
# Header
self.timestamp = unpack[1]
self.event_type = byte2int(unpack[2])
self.server_id = unpack[3]
self.event_size = unpack[4]
# position of the next event
self.log_pos = unpack[5]
self.flags = unpack[6]
self.file_name_binlog = file_name_binlog #DNX-GET-LINE
self.line_binlog = self.log_pos #DNX-GET-LINE
# MySQL 5.6 and more if binlog-checksum = CRC32
if use_checksum:
event_size_without_header = self.event_size - 23
else:
event_size_without_header = self.event_size - 19
self.event = None
event_class = self.__event_map.get(self.event_type,
event.NotImplementedEvent)
if event_class not in allowed_events:
return
self.event = event_class(
self,
event_size_without_header,
table_map,
ctl_connection,
file_name_binlog=self.file_name_binlog, #DNX-GET-LINE
line_binlog=self.line_binlog, #DNX-GET-LINE
only_tables=only_tables,
ignored_tables=ignored_tables,
only_schemas=only_schemas,
ignored_schemas=ignored_schemas,
freeze_schema=freeze_schema,
fail_on_table_metadata_unavailable=
fail_on_table_metadata_unavailable)
if self.event._processed == False:
self.event = None
def __init__(self, from_packet, table_map, ctl_connection):
if not from_packet.is_ok_packet():
raise ValueError('Cannot create ' + str(self.__class__.__name__)
+ ' object from invalid packet type')
self.read_bytes = 0 #-1 because we ignore the ok byte
self.__data_buffer = b'' #Used when we want to override a value in the data buffer
# Ok Value
self.packet = from_packet
self.packet.advance(1)
self.charset = ctl_connection.charset
# Header
self.timestamp = struct.unpack('<I', self.packet.read(4))[0]
self.event_type = byte2int(self.packet.read(1))
self.server_id = struct.unpack('<I', self.packet.read(4))[0]
self.event_size = struct.unpack('<I', self.packet.read(4))[0]
# position of the next event
self.log_pos = struct.unpack('<I', self.packet.read(4))[0]
self.flags = self.flags = struct.unpack('<H', self.packet.read(2))[0]
event_size_without_header = self.event_size - 19
try:
event_class = self.__event_map[self.event_type]
except KeyError:
raise NotImplementedError("Unknown MySQL bin log event type: " + hex(self.event_type))
self.event = event_class(self, event_size_without_header, table_map, ctl_connection)
def __init__(self, from_packet, event_size, table_map, ctl_connection):
super(GtidEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection)
self.commit_flag = byte2int(self.packet.read(1)) == 1
self.sid = self.packet.read(16)
self.gno = struct.unpack('<Q', self.packet.read(8))[0]
def __init__(self, from_packet, event_size, table_map, ctl_connection):
super(GtidEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection)
self.commit_flag = byte2int(self.packet.read(1)) == 1
self.sid = self.packet.read(16)
self.gno = struct.unpack('<Q', self.packet.read(8))[0]
def __init__(self, from_packet, table_map, ctl_connection):
if not from_packet.is_ok_packet():
raise ValueError('Cannot create ' + str(self.__class__.__name__) +
' object from invalid packet type')
self.read_bytes = 0 #-1 because we ignore the ok byte
self.__data_buffer = b'' #Used when we want to override a value in the data buffer
# Ok Value
self.packet = from_packet
self.packet.advance(1)
self.charset = ctl_connection.charset
# Header
self.timestamp = struct.unpack('<I', self.packet.read(4))[0]
self.event_type = byte2int(self.packet.read(1))
self.server_id = struct.unpack('<I', self.packet.read(4))[0]
self.event_size = struct.unpack('<I', self.packet.read(4))[0]
# position of the next event
self.log_pos = struct.unpack('<I', self.packet.read(4))[0]
self.flags = self.flags = struct.unpack('<H', self.packet.read(2))[0]
event_size_without_header = self.event_size - 19
try:
event_class = self.__event_map[self.event_type]
except KeyError:
raise NotImplementedError("Unknown MySQL bin log event type: " +
hex(self.event_type))
self.event = event_class(self, event_size_without_header, table_map,
ctl_connection)
def _get_server_information(self):
i = 0
packet = yield from self._read_packet()
data = packet.get_all_data()
# logger.debug(dump_packet(data))
self.protocol_version = byte2int(data[i:i + 1])
i += 1
server_end = data.find(b'\0', i)
self.server_version = data[i:server_end].decode('latin1')
i = server_end + 1
self.server_thread_id = struct.unpack('<I', data[i:i + 4])
i += 4
self.salt = data[i:i + 8]
i += 9 # 8 + 1(filler)
self.server_capabilities = struct.unpack('<H', data[i:i + 2])[0]
i += 2
if len(data) >= i + 6:
lang, stat, cap_h, salt_len = struct.unpack('<BHHB', data[i:i + 6])
i += 6
self.server_language = lang
self.server_charset = charset_by_id(lang).name
self.server_status = stat
# logger.debug("server_status: %s" % _convert_to_str(stat))
self.server_capabilities |= cap_h << 16
# logger.debug("salt_len: %s" % _convert_to_str(salt_len))
salt_len = max(12, salt_len - 9)
# reserved
i += 10
if len(data) >= i + salt_len:
# salt_len includes auth_plugin_data_part_1 and filler
self.salt += data[i:i + salt_len]
i += salt_len
i += 1
# AUTH PLUGIN NAME may appear here.
if self.server_capabilities & CLIENT.PLUGIN_AUTH and len(data) >= i:
# Due to Bug#59453 the auth-plugin-name is missing the terminating
# NUL-char in versions prior to 5.5.10 and 5.6.2.
# ref: https://dev.mysql.com/doc/internals/en/
# connection-phase-packets.html#packet-Protocol::Handshake
# didn't use version checks as mariadb is corrected and reports
# earlier than those two.
server_end = data.find(b'\0', i)
if server_end < 0: # pragma: no cover - very specific upstream bug
# not found \0 and last field so take it all
self._auth_plugin_name = data[i:].decode('latin1')
else:
self._auth_plugin_name = data[i:server_end].decode('latin1')
def __parse_field_descriptor(self, encoding):
"""Parse the 'Field Descriptor' (Metadata) packet.
This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0).
"""
self.catalog = self.read_length_coded_string()
self.db = self.read_length_coded_string()
self.table_name = self.read_length_coded_string().decode(encoding)
self.org_table = self.read_length_coded_string().decode(encoding)
self.name = self.read_length_coded_string().decode(encoding)
self.org_name = self.read_length_coded_string().decode(encoding)
self.advance(1) # non-null filler
self.charsetnr = struct.unpack('<H', self.read(2))[0]
self.length = struct.unpack('<I', self.read(4))[0]
self.type_code = byte2int(self.read(1))
self.flags = struct.unpack('<H', self.read(2))[0]
self.scale = byte2int(self.read(1)) # "decimals"
self.advance(2) # filler (always 0x00)
def __parse_field_descriptor(self, encoding):
"""Parse the 'Field Descriptor' (Metadata) packet.
This is compatible with MySQL 4.1+ (not compatible with MySQL 4.0).
"""
self.catalog = self.read_length_coded_string()
self.db = self.read_length_coded_string()
self.table_name = self.read_length_coded_string().decode(encoding)
self.org_table = self.read_length_coded_string().decode(encoding)
self.name = self.read_length_coded_string().decode(encoding)
self.org_name = self.read_length_coded_string().decode(encoding)
self.advance(1) # non-null filler
self.charsetnr = struct.unpack('<H', self.read(2))[0]
self.length = struct.unpack('<I', self.read(4))[0]
self.type_code = byte2int(self.read(1))
self.flags = struct.unpack('<H', self.read(2))[0]
self.scale = byte2int(self.read(1)) # "decimals"
self.advance(2) # filler (always 0x00)
def _get_server_information(self):
i = 0
packet = yield from self._read_packet()
data = packet.get_all_data()
# logger.debug(dump_packet(data))
self.protocol_version = byte2int(data[i:i + 1])
i += 1
server_end = data.find(b'\0', i)
self.server_version = data[i:server_end].decode('latin1')
i = server_end + 1
self.server_thread_id = struct.unpack('<I', data[i:i + 4])
i += 4
self.salt = data[i:i + 8]
i += 9 # 8 + 1(filler)
self.server_capabilities = struct.unpack('<H', data[i:i + 2])[0]
i += 2
if len(data) >= i + 6:
lang, stat, cap_h, salt_len = struct.unpack('<BHHB', data[i:i + 6])
i += 6
self.server_language = lang
self.server_charset = charset_by_id(lang).name
self.server_status = stat
# logger.debug("server_status: %s" % _convert_to_str(stat))
self.server_capabilities |= cap_h << 16
# logger.debug("salt_len: %s" % _convert_to_str(salt_len))
salt_len = max(12, salt_len - 9)
# reserved
i += 10
if len(data) >= i + salt_len:
# salt_len includes auth_plugin_data_part_1 and filler
self.salt += data[i:i + salt_len]
i += salt_len
i += 1
# AUTH PLUGIN NAME may appear here.
if self.server_capabilities & CLIENT.PLUGIN_AUTH and len(data) >= i:
# Due to Bug#59453 the auth-plugin-name is missing the terminating
# NUL-char in versions prior to 5.5.10 and 5.6.2.
# ref: https://dev.mysql.com/doc/internals/en/
# connection-phase-packets.html#packet-Protocol::Handshake
# didn't use version checks as mariadb is corrected and reports
# earlier than those two.
server_end = data.find(b'\0', i)
if server_end < 0: # pragma: no cover - very specific upstream bug
# not found \0 and last field so take it all
self._auth_plugin_name = data[i:].decode('latin1')
else:
self._auth_plugin_name = data[i:server_end].decode('latin1')
def __init__(self, from_packet, table_map, ctl_connection, use_checksum,
allowed_events,
only_tables,
ignored_tables,
only_schemas,
ignored_schemas,
freeze_schema,
fail_on_table_metadata_unavailable):
# -1 because we ignore the ok byte
self.read_bytes = 0
# Used when we want to override a value in the data buffer
self.__data_buffer = b''
self.packet = from_packet
self.charset = ctl_connection.charset
# OK value
# timestamp
# event_type
# server_id
# log_pos
# flags
unpack = struct.unpack('<cIcIIIH', self.packet.read(20))
# Header
self.timestamp = unpack[1]
self.event_type = byte2int(unpack[2])
self.server_id = unpack[3]
self.event_size = unpack[4]
# position of the next event
self.log_pos = unpack[5]
self.flags = unpack[6]
# MySQL 5.6 and more if binlog-checksum = CRC32
if use_checksum:
event_size_without_header = self.event_size - 23
else:
event_size_without_header = self.event_size - 19
self.event = None
event_class = self.__event_map.get(self.event_type, event.NotImplementedEvent)
if event_class not in allowed_events:
return
self.event = event_class(self, event_size_without_header, table_map,
ctl_connection,
only_tables=only_tables,
ignored_tables=ignored_tables,
only_schemas=only_schemas,
ignored_schemas=ignored_schemas,
freeze_schema=freeze_schema,
fail_on_table_metadata_unavailable=fail_on_table_metadata_unavailable)
if self.event._processed == False:
self.event = None
def __init__(self, header):
'''Initialize the Event with the event header'''
unpacked = struct.unpack('<IcIIIH', header)
self.timestamp = unpacked[0]
self.event_type = byte2int(unpacked[1])
self.server_id = unpacked[2]
self.event_size = unpacked[3]
self.log_pos = unpacked[4]
self.flags = unpacked[5]
self.body = None
self.pos = None
def __init__(self, header):
'''Initialize the Event with the event header'''
unpacked = struct.unpack('<IcIIIH', header)
self.timestamp = unpacked[0]
self.event_type = byte2int(unpacked[1])
self.server_id = unpacked[2]
self.event_size = unpacked[3]
self.log_pos = unpacked[4]
self.flags = unpacked[5]
self.body = None
self.pos = None
def read_variable_length_string(self):
"""Read a variable length string where the first 1-5 bytes stores the
length of the string.
For each byte, the first bit being high indicates another byte must be
read.
"""
byte = 0x80
length = 0
bits_read = 0
while byte & 0x80 != 0:
byte = byte2int(self.read(1))
length = length | ((byte & 0x7f) << bits_read)
bits_read = bits_read + 7
return self.read(length)
def read_variable_length_string(self):
"""Read a variable length string where the first 1-5 bytes stores the
length of the string.
For each byte, the first bit being high indicates another byte must be
read.
"""
byte = 0x80
length = 0
bits_read = 0
while byte & 0x80 != 0:
byte = byte2int(self.read(1))
length = length | ((byte & 0x7f) << bits_read)
bits_read = bits_read + 7
return self.read(length)
def __init__(self, from_packet, event_size, table_map, ctl_connection):
super(QueryEvent, self).__init__(from_packet, event_size, table_map, ctl_connection)
# Post-header
self.slave_proxy_id = self.packet.read_uint32()
self.execution_time = self.packet.read_uint32()
self.schema_length = byte2int(self.packet.read(1))
self.error_code = self.packet.read_uint16()
self.status_vars_length = self.packet.read_uint16()
# Payload
self.status_vars = self.packet.read(self.status_vars_length)
self.schema = self.packet.read(self.schema_length)
self.packet.advance(1)
self.query = self.packet.read(event_size - 13 - self.status_vars_length - self.schema_length - 1).decode()
def read_length_coded_binary(self):
"""Read a 'Length Coded Binary' number from the data buffer.
Length coded numbers can be anywhere from 1 to 9 bytes depending
on the value of the first byte.
From PyMYSQL source code
"""
c = byte2int(self.read(1))
if c == NULL_COLUMN:
return None
if c < UNSIGNED_CHAR_COLUMN:
return c
elif c == UNSIGNED_SHORT_COLUMN:
return self.unpack_uint16(self.read(UNSIGNED_SHORT_LENGTH))
elif c == UNSIGNED_INT24_COLUMN:
return self.unpack_int24(self.read(UNSIGNED_INT24_LENGTH))
elif c == UNSIGNED_INT64_COLUMN:
return self.unpack_int64(self.read(UNSIGNED_INT64_LENGTH))
def __init__(self, from_packet, event_size, table_map, ctl_connection):
super(QueryEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection)
# Post-header
self.slave_proxy_id = self.packet.read_uint32()
self.execution_time = self.packet.read_uint32()
self.schema_length = byte2int(self.packet.read(1))
self.error_code = self.packet.read_uint16()
self.status_vars_length = self.packet.read_uint16()
# Payload
self.status_vars = self.packet.read(self.status_vars_length)
self.schema = self.packet.read(self.schema_length)
self.packet.advance(1)
self.query = self.packet.read(event_size - 13 -
self.status_vars_length -
self.schema_length - 1).decode()
def read_length_coded_binary(self):
"""Read a 'Length Coded Binary' number from the data buffer.
Length coded numbers can be anywhere from 1 to 9 bytes depending
on the value of the first byte.
From PyMYSQL source code
"""
c = byte2int(self.read(1))
if c == NULL_COLUMN:
return None
if c < UNSIGNED_CHAR_COLUMN:
return c
elif c == UNSIGNED_SHORT_COLUMN:
return self.unpack_uint16(self.read(UNSIGNED_SHORT_LENGTH))
elif c == UNSIGNED_INT24_COLUMN:
return self.unpack_int24(self.read(UNSIGNED_INT24_LENGTH))
elif c == UNSIGNED_INT64_COLUMN:
return self.unpack_int64(self.read(UNSIGNED_INT64_LENGTH))
def read_binlog(skt):
while True:
packet = read_server_packet(skt)
sequenceId = getSequenceId(packet)
packetType = getType(packet)
# OK value
# timestamp
# event_type
# server_id
# log_pos
# flags
unpack = struct.unpack('<cIcIIIH', packet[6:26])
# Header
timestamp = unpack[1]
event_type = byte2int(unpack[2])
server_id = unpack[3]
event_size = unpack[4]
# position of the next event
log_pos = unpack[5]
flags = unpack[6]
print('timestamp', 'event_type', 'server_id', 'event_size', 'log_pos')
print(timestamp, event_type, server_id, event_size, log_pos)
dump_my_packet(packet)
if event_type == 16:
ack = SemiAck("mysql-bin.000001", log_pos)
ack.sequenceId = 0
packet = ack.toPacket()
send_socket(skt, packet)
if packetType == Flags.ERR:
buf = ERR.loadFromPacket(packet)
print("error:", buf.errorCode, buf.sqlState, buf.errorMessage)
skt.close()
exit(1)
break
if packetType == Flags.EOF or packetType == Flags.OK:
break
def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
super(QueryEvent, self).__init__(from_packet, event_size, table_map, ctl_connection, **kwargs)
# Post-header
self.slave_proxy_id = self.packet.read_uint32()
self.execution_time = self.packet.read_uint32()
self.schema_length = byte2int(self.packet.read(1))
self.error_code = self.packet.read_uint16()
self.status_vars_length = self.packet.read_uint16()
# Payload
self.status_vars = self.packet.read(self.status_vars_length)
self.schema = self.packet.read(self.schema_length)
self.packet.advance(1)
self.query = self.packet.read(event_size - 13 - self.status_vars_length - self.schema_length - 1)
try:
self.query = self.query.decode("utf-8")
except UnicodeDecodeError:
self.query = self.query.decode("latin-1")
def _get_server_information(self):
i = 0
packet = yield from self._read_packet()
data = packet.get_all_data()
# logger.debug(dump_packet(data))
self.protocol_version = byte2int(data[i:i + 1])
i += 1
server_end = data.find(int2byte(0), i)
self.server_version = data[i:server_end].decode('latin1')
i = server_end + 1
self.server_thread_id = struct.unpack('<I', data[i:i + 4])
i += 4
self.salt = data[i:i + 8]
i += 9 # 8 + 1(filler)
self.server_capabilities = struct.unpack('<H', data[i:i + 2])[0]
i += 2
if len(data) >= i + 6:
lang, stat, cap_h, salt_len = struct.unpack('<BHHB', data[i:i + 6])
i += 6
self.server_language = lang
self.server_charset = charset_by_id(lang).name
self.server_status = stat
# logger.debug("server_status: %s" % _convert_to_str(stat))
self.server_capabilities |= cap_h << 16
# logger.debug("salt_len: %s" % _convert_to_str(salt_len))
salt_len = max(12, salt_len - 9)
# reserved
i += 10
if len(data) >= i + salt_len:
# salt_len includes auth_plugin_data_part_1 and filler
self.salt += data[i:i + salt_len]
def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
super(QueryEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection, **kwargs)
self.__only_schemas = kwargs["only_schemas"]
# Post-header
self.slave_proxy_id = self.packet.read_uint32()
self.execution_time = self.packet.read_uint32()
self.schema_length = byte2int(self.packet.read(1))
self.error_code = self.packet.read_uint16()
self.status_vars_length = self.packet.read_uint16()
# Payload
self.status_vars = self.packet.read(self.status_vars_length)
self.schema = self.packet.read(self.schema_length)
self.packet.advance(1)
self.query = self.packet.read(event_size - 13 - self.status_vars_length
- self.schema_length - 1).decode("utf-8")
#string[EOF] query
self.__hashes = {}
def _get_server_information(self):
i = 0
packet = yield from self._read_packet()
data = packet.get_all_data()
# logger.debug(dump_packet(data))
self.protocol_version = byte2int(data[i:i + 1])
i += 1
server_end = data.find(int2byte(0), i)
self.server_version = data[i:server_end].decode('latin1')
i = server_end + 1
self.server_thread_id = struct.unpack('<I', data[i:i + 4])
i += 4
self.salt = data[i:i + 8]
i += 9 # 8 + 1(filler)
self.server_capabilities = struct.unpack('<H', data[i:i + 2])[0]
i += 2
if len(data) >= i + 6:
lang, stat, cap_h, salt_len = struct.unpack('<BHHB', data[i:i + 6])
i += 6
self.server_language = lang
self.server_charset = charset_by_id(lang).name
self.server_status = stat
# logger.debug("server_status: %s" % _convert_to_str(stat))
self.server_capabilities |= cap_h << 16
# logger.debug("salt_len: %s" % _convert_to_str(salt_len))
salt_len = max(12, salt_len - 9)
# reserved
i += 10
if len(data) >= i + salt_len:
# salt_len includes auth_plugin_data_part_1 and filler
self.salt += data[i:i + salt_len]
def dump_packet(data): # pragma: no cover
def printable(data):
if 32 <= byte2int(data) < 127:
if isinstance(data, int):
return chr(data)
return data
return '.'
try:
print("packet length:", len(data))
for i in range(1, 7):
f = sys._getframe(i)
print("call[%d]: %s (line %d)" % (i, f.f_code.co_name, f.f_lineno))
print("-" * 66)
except ValueError:
pass
dump_data = [
data[i:i + 16] for i in range_type(0, min(len(data), 256), 16)
]
for d in dump_data:
print(' '.join("{:02X}".format(byte2int(x)) for x in d) + ' ' *
(16 - len(d)) + ' ' * 2 + ''.join(printable(x) for x in d))
print("-" * 66)
print()
def __init__(self, from_packet, event_size, table_map, ctl_connection,
**kwargs):
super(TableMapEvent, self).__init__(from_packet, event_size, table_map,
ctl_connection, **kwargs)
self.__only_tables = kwargs["only_tables"]
self.__ignored_tables = kwargs["ignored_tables"]
self.__only_schemas = kwargs["only_schemas"]
self.__ignored_schemas = kwargs["ignored_schemas"]
self.__freeze_schema = kwargs["freeze_schema"]
# Post-Header
self.table_id = self._read_table_id()
if self.table_id in table_map and self.__freeze_schema:
self._processed = False
return
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
if self.__only_tables is not None and self.table not in self.__only_tables:
self._processed = False
return
elif self.__ignored_tables is not None and self.table in self.__ignored_tables:
self._processed = False
return
if self.__only_schemas is not None and self.schema not in self.__only_schemas:
self._processed = False
return
elif self.__ignored_schemas is not None and self.schema in self.__ignored_schemas:
self._processed = False
return
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self._ctl_connection._get_table_information(
self.schema, self.table)
if len(self.column_schemas) != 0:
# Read columns meta data
column_types = list(self.packet.read(self.column_count))
self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
try:
column_schema = self.column_schemas[i]
except IndexError:
# this a dirty hack to prevent row events containing columns which have been dropped prior
# to pymysqlreplication start, but replayed from binlog from blowing up the service.
# TODO: this does not address the issue if the column other than the last one is dropped
column_schema = {
'COLUMN_NAME': '__dropped_col_{i}__'.format(i=i),
'COLLATION_NAME': None,
'CHARACTER_SET_NAME': None,
'COLUMN_COMMENT': None,
'COLUMN_TYPE':
'BLOB', # we don't know what it is, so let's not do anything with it.
'COLUMN_KEY': '',
}
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
self.table_obj = Table(self.column_schemas, self.table_id, self.schema,
self.table, self.columns)
def _get_server_information(self):
i = 0
# read row packet
raw_data = self._read_packet_raw()
# with ignoring 4-byte header
packet = MysqlPacket(raw_data[4:], self.encoding)
packet.check_error()
data = packet.get_all_data()
self.protocol_version = byte2int(data[i:i + 1])
i += 1
server_end = data.find(b'\0', i)
self.server_version = data[i:server_end].decode('latin1')
i = server_end + 1
self.server_thread_id = struct.unpack('<I', data[i:i + 4])
i += 4
self.salt = data[i:i + 8]
i += 9 # 8 + 1(filler)
self.server_capabilities = struct.unpack('<H', data[i:i + 2])[0]
i += 2
if len(data) >= i + 6:
lang, stat, cap_h, salt_len = struct.unpack('<BHHB', data[i:i + 6])
i += 6
# TODO: deprecate server_language and server_charset.
# mysqlclient-python doesn't provide it.
self.server_language = lang
try:
self.server_charset = charset_by_id(lang).name
except KeyError:
# unknown collation
self.server_charset = None
self.server_status = stat
if DEBUG: print("server_status: %x" % stat)
self.server_capabilities |= cap_h << 16
if DEBUG: print("salt_len:", salt_len)
salt_len = max(12, salt_len - 9)
# reserved
i += 10
if len(data) >= i + salt_len:
# salt_len includes auth_plugin_data_part_1 and filler
self.salt += data[i:i + salt_len]
i += salt_len
i += 1
# AUTH PLUGIN NAME may appear here.
if self.server_capabilities & CLIENT.PLUGIN_AUTH and len(data) >= i:
# Due to Bug#59453 the auth-plugin-name is missing the terminating
# NUL-char in versions prior to 5.5.10 and 5.6.2.
# ref: https://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::Handshake
# didn't use version checks as mariadb is corrected and reports
# earlier than those two.
server_end = data.find(b'\0', i)
if server_end < 0: # pragma: no cover - very specific upstream bug
# not found \0 and last field so take it all
self._auth_plugin_name = data[i:].decode('latin1')
else:
self._auth_plugin_name = data[i:server_end].decode('latin1')
return raw_data
def fetchone(self):
while True:
if not self._file:
self.__connect_to_stream()
if not self.__connected_ctl and self._ctl_connection_settings:
self.__connect_to_ctl()
# read pkt
pkt = StringIOAdvance()
# headerlength 19
header = self._file.read(19)
if not header:
break
unpacked = struct.unpack('<IcIIIH', header)
timestamp = unpacked[0]
event_type = byte2int(unpacked[1])
server_id = unpacked[2]
event_size = unpacked[3]
log_pos = unpacked[4]
flags = unpacked[5]
body = self._file.read(event_size - 19)
pkt.write('0')
pkt.write(header)
pkt.write(body)
pkt.seek(0)
binlog_event = BinLogPacketWrapper(pkt, self.table_map,
self._ctl_connection,
self.__use_checksum,
self.__allowed_events_in_packet,
self.__only_tables,
self.__ignored_tables,
self.__only_schemas,
self.__ignored_schemas,
self.__freeze_schema,
self.__fail_on_table_metadata_unavailable)
if not binlog_event.event:
continue
if binlog_event.event_type == ROTATE_EVENT:
self.log_pos = binlog_event.event.position
self.log_file = binlog_event.event.next_binlog
# Table Id in binlog are NOT persistent in MySQL - they are in-memory identifiers
# that means that when MySQL master restarts, it will reuse same table id for different tables
# which will cause errors for us since our in-memory map will try to decode row data with
# wrong table schema.
# The fix is to rely on the fact that MySQL will also rotate to a new binlog file every time it
# restarts. That means every rotation we see *could* be a sign of restart and so potentially
# invalidates all our cached table id to schema mappings. This means we have to load them all
# again for each logfile which is potentially wasted effort but we can't really do much better
# without being broken in restart case
self.table_map = {}
elif binlog_event.log_pos:
self.log_pos = binlog_event.log_pos
# This check must not occur before clearing the ``table_map`` as a
# result of a RotateEvent.
#
# The first RotateEvent in a binlog file has a timestamp of
# zero. If the server has moved to a new log and not written a
# timestamped RotateEvent at the end of the previous log, the
# RotateEvent at the beginning of the new log will be ignored
# if the caller provided a positive ``skip_to_timestamp``
# value. This will result in the ``table_map`` becoming
# corrupt.
#
# https://dev.mysql.com/doc/internals/en/event-data-for-specific-event-types.html
# From the MySQL Internals Manual:
#
# ROTATE_EVENT is generated locally and written to the binary
# log on the master. It is written to the relay log on the
# slave when FLUSH LOGS occurs, and when receiving a
# ROTATE_EVENT from the master. In the latter case, there
# will be two rotate events in total originating on different
# servers.
#
# There are conditions under which the terminating
# log-rotation event does not occur. For example, the server
# might crash.
if self.skip_to_timestamp and binlog_event.timestamp < self.skip_to_timestamp:
continue
if binlog_event.event_type == TABLE_MAP_EVENT and \
binlog_event.event is not None:
self.table_map[binlog_event.event.table_id] = \
binlog_event.event.get_table()
# event is none if we have filter it on packet level
# we filter also not allowed events
if binlog_event.event is None or (binlog_event.event.__class__ not in self.__allowed_events):
continue
return binlog_event.event
def printable(data):
if 32 <= byte2int(data) < 127:
if isinstance(data, int):
return chr(data)
return data
return '.'
def __init__(self, from_packet, event_size, table_map, ctl_connection, **kwargs):
super(TableMapEvent, self).__init__(from_packet, event_size,
table_map, ctl_connection, **kwargs)
self.__only_tables = kwargs["only_tables"]
self.__ignored_tables = kwargs["ignored_tables"]
self.__only_schemas = kwargs["only_schemas"]
self.__ignored_schemas = kwargs["ignored_schemas"]
self.__freeze_schema = kwargs["freeze_schema"]
# Post-Header
self.table_id = self._read_table_id()
if self.table_id in table_map and self.__freeze_schema:
self._processed = False
return
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
if self.__only_tables is not None and self.table not in self.__only_tables:
self._processed = False
return
elif self.__ignored_tables is not None and self.table in self.__ignored_tables:
self._processed = False
return
if self.__only_schemas is not None and self.schema not in self.__only_schemas:
self._processed = False
return
elif self.__ignored_schemas is not None and self.schema in self.__ignored_schemas:
self._processed = False
return
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self._ctl_connection._get_table_information(self.schema, self.table)
if len(self.column_schemas) != 0:
# Read columns meta data
column_types = list(self.packet.read(self.column_count))
self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
try:
column_schema = self.column_schemas[i]
except IndexError:
# this a dirty hack to prevent row events containing columns which have been dropped prior
# to pymysqlreplication start, but replayed from binlog from blowing up the service.
# TODO: this does not address the issue if the column other than the last one is dropped
column_schema = {
'COLUMN_NAME': '__dropped_col_{i}__'.format(i=i),
'COLLATION_NAME': None,
'CHARACTER_SET_NAME': None,
'COLUMN_COMMENT': None,
'COLUMN_TYPE': 'BLOB', # we don't know what it is, so let's not do anything with it.
'COLUMN_KEY': '',
}
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
self.table_obj = Table(self.column_schemas, self.table_id, self.schema,
self.table, self.columns)
def __init__(
self,
from_packet,
event_size,
table_map,
ctl_connection,
file_name_binlog, # DNX-GET-LINE
line_binlog, # DNX-GET-LINE
**kwargs):
super(TableMapEvent, self).__init__(
from_packet,
event_size,
table_map,
ctl_connection,
file_name_binlog, # DNX-GET-LINE
line_binlog, # DNX-GET-LINE
**kwargs)
self.__only_tables = kwargs["only_tables"]
self.__ignored_tables = kwargs["ignored_tables"]
self.__only_schemas = kwargs["only_schemas"]
self.__ignored_schemas = kwargs["ignored_schemas"]
self.__freeze_schema = kwargs["freeze_schema"]
self.file_name_binlog = file_name_binlog # DNX-GET-LINE
self.line_binlog = line_binlog # DNX-GET-LINE
# Post-Header
self.table_id = self._read_table_id()
if self.table_id in table_map and self.__freeze_schema:
self._processed = False
return
self.flags = struct.unpack('<H', self.packet.read(2))[0]
# Payload
self.schema_length = byte2int(self.packet.read(1))
self.schema = self.packet.read(self.schema_length).decode()
self.packet.advance(1)
self.table_length = byte2int(self.packet.read(1))
self.table = self.packet.read(self.table_length).decode()
if self.__only_tables is not None and self.table not in self.__only_tables:
self._processed = False
return
elif self.__ignored_tables is not None and self.table in self.__ignored_tables:
self._processed = False
return
if self.__only_schemas is not None and self.schema not in self.__only_schemas:
self._processed = False
return
elif self.__ignored_schemas is not None and self.schema in self.__ignored_schemas:
self._processed = False
return
self.packet.advance(1)
self.column_count = self.packet.read_length_coded_binary()
self.columns = []
if self.table_id in table_map:
self.column_schemas = table_map[self.table_id].column_schemas
else:
self.column_schemas = self._ctl_connection._get_table_information(
self.schema, self.table)
ordinal_pos_loc = 0
if len(self.column_schemas) != 0:
# Read columns meta data
column_types = list(self.packet.read(self.column_count))
self.packet.read_length_coded_binary()
for i in range(0, len(column_types)):
column_type = column_types[i]
try:
column_schema = self.column_schemas[ordinal_pos_loc]
# only acknowledge the column definition if the iteration matches with ordinal position of
# the column. this helps in maintaining support for restricted columnar access
if i != (column_schema['ORDINAL_POSITION'] - 1):
# raise IndexError to follow the workflow of dropping columns which are not matching the
# underlying table schema
raise IndexError
ordinal_pos_loc += 1
except IndexError:
# this a dirty hack to prevent row events containing columns which have been dropped prior
# to pymysqlreplication start, but replayed from binlog from blowing up the service.
# TODO: this does not address the issue if the column other than the last one is dropped
column_schema = {
'COLUMN_NAME': '__dropped_col_{i}__'.format(i=i),
'COLLATION_NAME': None,
'CHARACTER_SET_NAME': None,
'COLUMN_COMMENT': None,
'COLUMN_TYPE':
'BLOB', # we don't know what it is, so let's not do anything with it.
'COLUMN_KEY': '',
}
col = Column(byte2int(column_type), column_schema, from_packet)
self.columns.append(col)
self.table_obj = Table(self.column_schemas, self.table_id, self.schema,
self.table, self.columns)
