def update_restore_log(instance, row_id, params):
try:
conn = mysql_lib.connect_mysql(instance)
except Exception as e:
log.warning("Unable to connect to master to log "
"our progress: {e}. Attempting to "
"continue with restore anyway.".format(e=e))
return
updates_fields = []
if 'finished_at' in params:
updates_fields.append('finished_at=NOW()')
if 'restore_status' in params:
updates_fields.append('restore_status=%(restore_status)s')
if 'status_message' in params:
updates_fields.append('status_message=%(status_message)s')
if 'replication' in params:
updates_fields.append('replication=%(replication)s')
if 'zookeeper' in params:
updates_fields.append('zookeeper=%(zookeeper)s')
if 'finished_at' in params:
updates_fields.append('finished_at=NOW()')
sql = ("UPDATE test.xb_restore_status SET "
+ ', '.join(updates_fields) +
" WHERE id = %(row_id)s")
params['row_id'] = row_id
cursor = conn.cursor()
cursor.execute(sql, params)
log.info(cursor._executed)
cursor.close()
conn.commit()
conn.close()
def update_restore_log(instance, row_id, params):
try:
conn = mysql_lib.connect_mysql(instance)
except Exception as e:
log.warning("Unable to connect to master to log "
"our progress: {e}. Attempting to "
"continue with restore anyway.".format(e=e))
return
updates_fields = []
if 'finished_at' in params:
updates_fields.append('finished_at=NOW()')
if 'restore_status' in params:
updates_fields.append('restore_status=%(restore_status)s')
if 'status_message' in params:
updates_fields.append('status_message=%(status_message)s')
if 'replication' in params:
updates_fields.append('replication=%(replication)s')
if 'zookeeper' in params:
updates_fields.append('zookeeper=%(zookeeper)s')
if 'finished_at' in params:
updates_fields.append('finished_at=NOW()')
sql = ("UPDATE test.xb_restore_status SET " + ', '.join(updates_fields) +
" WHERE id = %(row_id)s")
params['row_id'] = row_id
cursor = conn.cursor()
cursor.execute(sql, params)
log.info(cursor._executed)
cursor.close()
conn.commit()
conn.close()
def get_age_last_restore(replica_set):
""" Determine age of last successful backup restore
Args:
replica_set - A MySQL replica set
Returns - A tuple of age in days and name of a replica set. This is done
to make it easy to use multiprocessing.
"""
zk = host_utils.MysqlZookeeper()
today = datetime.date.today()
age = None
master = zk.get_mysql_instance_from_replica_set(replica_set)
try:
conn = mysql_lib.connect_mysql(master)
cursor = conn.cursor()
sql = ("SELECT restore_file "
"FROM test.xb_restore_status "
"WHERE restore_status='OK' "
"ORDER BY finished_at DESC "
"LIMIT 10")
# The most recent restore is not always using the newest restore file
# so we will just grab the 10 most recent.
cursor.execute(sql)
restores = cursor.fetchall()
except Exception as e:
log.error(e)
return
for restore in restores:
_, creation = get_metadata_from_backup_file(restore['restore_file'])
if age is None or (today - creation).days < age:
age = (today - creation).days
return (age, replica_set)
def get_age_last_restore(replica_set):
""" Determine age of last successful backup restore
Args:
replica_set - A MySQL replica set
Returns - A tuple of age in days and name of a replica set. This is done
to make it easy to use multiprocessing.
"""
zk = host_utils.MysqlZookeeper()
today = datetime.date.today()
age = None
master = zk.get_mysql_instance_from_replica_set(replica_set)
try:
conn = mysql_lib.connect_mysql(master)
cursor = conn.cursor()
sql = ("SELECT restore_file "
"FROM test.xb_restore_status "
"WHERE restore_status='OK' "
" AND finished_at > NOW() - INTERVAL {LARGE_RESTORE_AGE} DAY "
"ORDER BY finished_at DESC "
"LIMIT 1"
).format(LARGE_RESTORE_AGE=LARGE_RESTORE_AGE)
cursor.execute(sql)
restore = cursor.fetchone()
_, creation = get_metadata_from_backup_file(restore['restore_file'])
age = (today - creation).days
if age is None:
log.info('No restore table entries, setting restore age for'
'replica set {rs} to {big}'
''.format(rs=replica_set,
big=LARGE_RESTORE_AGE))
age = LARGE_RESTORE_AGE
except Exception as e:
log.error(': '.join([replica_set, e.__str__()]))
if e.args[0] == mysql_lib.MYSQL_ERROR_NO_SUCH_TABLE:
log.info('No restore table, setting restore age for replica set '
'{rs} to {big}'
''.format(rs=replica_set,
big=LARGE_RESTORE_AGE))
age = LARGE_RESTORE_AGE
elif e.args[0] == mysql_lib.MYSQL_ERROR_CONN_HOST_ERROR:
log.info('Could not connect, setting restore age for replica {} '
'to 0'
''.format(replica_set))
age = 0
else:
log.info('Unhandled exception, setting restore age for replica '
'{rs} to {big}'.format(rs=replica_set,
big=LARGE_RESTORE_AGE))
age = LARGE_RESTORE_AGE
return (age, replica_set)
def quick_test_replication(instance):
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
cursor.execute('START SLAVE')
time.sleep(2)
ss = mysql_lib.get_slave_status(conn)
if ss['Slave_IO_Running'] == 'No':
raise Exception('Replication [IO THREAD] failed '
'to start: {e}'.format(e=ss['Last_IO_Error']))
if ss['Slave_SQL_Running'] == 'No':
raise Exception('Replication [SQL THREAD] failed '
'to start: {e}'.format(e=ss['Last_SQL_Error']))
def get_most_recent_restore(instance):
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
sql = ("SELECT * "
"FROM test.xb_restore_status "
"WHERE restore_status='OK' ")
try:
cursor.execute(sql)
except Exception as e:
print ("UNKNOWN: Cannot query restore status table: {e}".format(e=e))
sys.exit(3)
return cursor.fetchall()
def get_most_recent_restore(instance):
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
sql = ("SELECT * "
"FROM test.xb_restore_status "
"WHERE restore_status='OK' ")
try:
cursor.execute(sql)
except Exception as e:
print("UNKNOWN: Cannot query restore status table: {e}".format(e=e))
sys.exit(3)
return cursor.fetchall()
def quick_test_replication(instance):
conn = mysql_lib.connect_mysql(instance)
cursor = conn.cursor()
cursor.execute('START SLAVE')
time.sleep(2)
ss = mysql_lib.get_slave_status(conn)
if ss['Slave_IO_Running'] == 'No':
raise Exception('Replication [IO THREAD] failed '
'to start: {e}'.format(e=ss['Last_IO_Error']))
if ss['Slave_SQL_Running'] == 'No':
raise Exception('Replication [SQL THREAD] failed '
'to start: {e}'.format(e=ss['Last_SQL_Error']))
def start_restore_log(instance, params):
""" Create a record in xb_restore_status at the start of a restore
Args:
instance - A hostaddr for where to log to
params - Parameters to be used in the INSERT
Returns:
The row_id of the created restore log entry
"""
try:
conn = mysql_lib.connect_mysql(instance)
except Exception as e:
log.warning("Unable to connect to master to log "
"our progress: {e}. Attempting to "
"continue with restore anyway.".format(e=e))
return None
if not mysql_lib.does_table_exist(instance, 'test', 'xb_restore_status'):
create_status_table(conn)
sql = ("REPLACE INTO test.xb_restore_status "
"SET "
"restore_source = %(restore_source)s, "
"restore_type = 's3', "
"restore_file = %(restore_file)s, "
"restore_destination = %(source_instance)s, "
"restore_date = %(restore_date)s, "
"restore_port = %(restore_port)s, "
"replication = %(replication)s, "
"zookeeper = %(zookeeper)s, "
"started_at = NOW()")
cursor = conn.cursor()
try:
cursor.execute(sql, params)
log.info(cursor._executed)
row_id = cursor.lastrowid
except Exception as e:
log.warning("Unable to log restore_status: {e}".format(e=e))
row_id = None
cursor.close()
conn.commit()
conn.close()
return row_id
