def print_rollback_sql(self, fin):
'''print rollback sql from tmpfile'''
with open(fin) as ftmp:
sleepInterval = 1000
i = 0
for line in reversed_lines(ftmp):
print line.rstrip()
if i >= sleepInterval:
print 'SELECT SLEEP(1);'
i = 0
else:
i += 1
def process_binlog(self):
stream = BinLogStreamReader(connection_settings=self.connectionSettings, server_id=self.serverId,
log_file=self.startFile, log_pos=self.startPos, only_schemas=self.only_schemas,
only_tables=self.only_tables, resume_stream=True)
cur = self.connection.cursor()
tmpFile = create_unique_file('%s.%s' % (self.connectionSettings['host'],self.connectionSettings['port'])) # to simplify code, we do not use file lock for tmpFile.
ftmp = open(tmpFile ,"w")
flagLastEvent = False
eStartPos, lastPos = stream.log_pos, stream.log_pos
try:
for binlogevent in stream:
if not self.stopnever:
if (stream.log_file == self.endFile and stream.log_pos == self.endPos) or (stream.log_file == self.eofFile and stream.log_pos == self.eofPos):
flagLastEvent = True
elif datetime.datetime.fromtimestamp(binlogevent.timestamp) < self.startTime:
if not (isinstance(binlogevent, RotateEvent) or isinstance(binlogevent, FormatDescriptionEvent)):
lastPos = binlogevent.packet.log_pos
continue
elif (stream.log_file not in self.binlogList) or (self.endPos and stream.log_file == self.endFile and stream.log_pos > self.endPos) or (stream.log_file == self.eofFile and stream.log_pos > self.eofPos) or (datetime.datetime.fromtimestamp(binlogevent.timestamp) >= self.stopTime):
break
# else:
# raise ValueError('unknown binlog file or position')
if isinstance(binlogevent, QueryEvent) and binlogevent.query == 'BEGIN':
eStartPos = lastPos
if isinstance(binlogevent, QueryEvent):
sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, flashback=self.flashback, nopk=self.nopk)
if sql:
print sql
elif isinstance(binlogevent, WriteRowsEvent) or isinstance(binlogevent, UpdateRowsEvent) or isinstance(binlogevent, DeleteRowsEvent):
for row in binlogevent.rows:
sql = concat_sql_from_binlogevent(cursor=cur, binlogevent=binlogevent, row=row , flashback=self.flashback, nopk=self.nopk, eStartPos=eStartPos)
if self.flashback:
ftmp.write(sql + '\n')
else:
print sql
if not (isinstance(binlogevent, RotateEvent) or isinstance(binlogevent, FormatDescriptionEvent)):
lastPos = binlogevent.packet.log_pos
if flagLastEvent:
break
ftmp.close()
if self.flashback:
with open(tmpFile) as ftmp:
for line in reversed_lines(ftmp):
print line.rstrip()
finally:
os.remove(tmpFile)
cur.close()
stream.close()
return True
def print_rollback_sql(self, filename):
"""print rollback sql from tmp_file"""
with open(filename, "rb") as f_tmp:
batch_size = 1000
i = 0
for line in reversed_lines(f_tmp):
print(line.rstrip())
if i >= batch_size:
i = 0
if self.back_interval:
print('SELECT SLEEP(%s);' % self.back_interval)
else:
i += 1
def process_binlog(self):
stream = BinLogStreamReader(connection_settings=self.conn_setting,
server_id=self.server_id,
log_file=self.start_file,
log_pos=self.start_pos,
only_schemas=self.only_schemas,
only_tables=self.only_tables,
resume_stream=True,
blocking=True)
flag_last_event = False
e_start_pos, last_pos = stream.log_pos, stream.log_pos
# to simplify code, we do not use flock for tmp_file.
tmp_file = create_unique_file(
'%s.%s' % (self.conn_setting['host'], self.conn_setting['port']))
try:
with temp_open(tmp_file, "w") as f_tmp, self.connection as cursor:
count = 0
for binlog_event in stream:
if count >= self.countnum:
break
if not self.stop_never:
try:
event_time = datetime.datetime.fromtimestamp(
binlog_event.timestamp)
except OSError:
event_time = datetime.datetime(1980, 1, 1, 0, 0)
if (stream.log_file == self.end_file and stream.log_pos == self.end_pos) or \
(stream.log_file == self.eof_file and stream.log_pos == self.eof_pos):
flag_last_event = True #解析道文件尾
elif event_time < self.start_time:
if not (isinstance(binlog_event, RotateEvent)
or isinstance(binlog_event,
FormatDescriptionEvent)):
last_pos = binlog_event.packet.log_pos
continue
elif (stream.log_file not in self.binlogList) or \
(self.end_pos and stream.log_file == self.end_file and stream.log_pos > self.end_pos) or \
(stream.log_file == self.eof_file and stream.log_pos > self.eof_pos) or \
(event_time >= self.stop_time):
break
# else:
# raise ValueError('unknown binlog file or position')
if isinstance(
binlog_event,
QueryEvent) and binlog_event.query == 'BEGIN':
e_start_pos = last_pos
if isinstance(binlog_event,
QueryEvent) and not self.only_dml:
sql = concat_sql_from_binlog_event(
cursor=cursor,
binlog_event=binlog_event,
flashback=self.flashback,
no_pk=self.no_pk)
if sql:
count = count + 1
self.sqllist.append(sql)
print(sql)
elif is_dml_event(binlog_event) and event_type(
binlog_event) in self.sql_type:
for row in binlog_event.rows:
sql = concat_sql_from_binlog_event(
cursor=cursor,
binlog_event=binlog_event,
no_pk=self.no_pk,
row=row,
flashback=self.flashback,
e_start_pos=e_start_pos)
if self.flashback:
f_tmp.write(sql + '\n')
else:
self.sqllist.append(sql)
print(sql)
count = count + 1
if not (isinstance(binlog_event, RotateEvent) or
isinstance(binlog_event, FormatDescriptionEvent)):
last_pos = binlog_event.packet.log_pos
if flag_last_event:
break
stream.close()
f_tmp.close()
if self.flashback:
with open(tmp_file) as ftmp:
for line in reversed_lines(ftmp):
self.sqllist.append(line.rstrip())
self.print_rollback_sql(filename=tmp_file)
finally:
os.remove(tmp_file)
return True