def protect_host(hostname, reason): """ Cause an host to not be acted on by the retirement queue Args: hostname - The hostname to protect reason - An explanation for why this host should not be retired dry_run - If set, don't modify state """ protecting_user = host_utils.get_user() if protecting_user == 'root': raise Exception('Can not modify retirement protection as root') reporting_conn = mysql_lib.get_mysqlops_connections() cursor = reporting_conn.cursor() sql = ("INSERT INTO mysqlops.retirement_protection " "SET " "hostname = %(hostname)s, " "reason = %(reason)s, " "protecting_user = %(protecting_user)s") cursor.execute(sql, { 'hostname': hostname, 'reason': reason, 'protecting_user': protecting_user }) reporting_conn.commit() log.info(cursor._executed)
def get_promotion_lock(replica_set): """ Take a promotion lock Args: replica_set - The replica set to take the lock against Returns: A unique identifer for the lock """ lock_identifier = str(uuid.uuid4()) log.info('Promotion lock identifier is ' '{lock_identifier}'.format(lock_identifier=lock_identifier)) conn = mysql_lib.get_mysqlops_connections() log.info('Releasing any expired locks') release_expired_promotion_locks(conn) log.info('Checking existing locks') check_promotion_lock(conn, replica_set) log.info('Taking lock against replica set: ' '{replica_set}'.format(replica_set=replica_set)) params = { 'lock': lock_identifier, 'localhost': host_utils.HOSTNAME, 'replica_set': replica_set, 'user': host_utils.get_user() } sql = ("INSERT INTO mysqlops.promotion_locks " "SET " "lock_identifier = %(lock)s, " "lock_active = 'active', " "created_at = NOW(), " "expires = NOW() + INTERVAL 12 HOUR, " "released = NULL, " "replica_set = %(replica_set)s, " "promoting_host = %(localhost)s, " "promoting_user = %(user)s ") cursor = conn.cursor() cursor.execute(sql, params) conn.commit() log.info(cursor._executed) return lock_identifier
def get_promotion_lock(replica_set): """ Take a promotion lock Args: replica_set - The replica set to take the lock against Returns: A unique identifer for the lock """ lock_identifier = str(uuid.uuid4()) log.info('Promotion lock identifier is ' '{lock_identifier}'.format(lock_identifier=lock_identifier)) conn = mysql_lib.get_mysqlops_connections() log.info('Releasing any expired locks') release_expired_promotion_locks(conn) log.info('Checking existing locks') check_promotion_lock(conn, replica_set) log.info('Taking lock against replica set: ' '{replica_set}'.format(replica_set=replica_set)) params = {'lock': lock_identifier, 'localhost': host_utils.HOSTNAME, 'replica_set': replica_set, 'user': host_utils.get_user()} sql = ("INSERT INTO mysqlops.promotion_locks " "SET " "lock_identifier = %(lock)s, " "lock_active = 'active', " "created_at = NOW(), " "expires = NOW() + INTERVAL 12 HOUR, " "released = NULL, " "replica_set = %(replica_set)s, " "promoting_host = %(localhost)s, " "promoting_user = %(user)s ") cursor = conn.cursor() cursor.execute(sql, params) conn.commit() log.info(cursor._executed) return lock_identifier
def protect_host(hostname, reason): """ Cause an host to not be acted on by the retirement queue Args: hostname - The hostname to protect reason - An explanation for why this host should not be retired dry_run - If set, don't modify state """ protecting_user = host_utils.get_user() if protecting_user == 'root': raise Exception('Can not modify retirement protection as root') reporting_conn = mysql_lib.get_mysqlops_connections() cursor = reporting_conn.cursor() sql = ("INSERT INTO mysqlops.retirement_protection " "SET " "hostname = %(hostname)s, " "reason = %(reason)s, " "protecting_user = %(protecting_user)s") cursor.execute(sql, {'hostname': hostname, 'reason': reason, 'protecting_user': protecting_user}) reporting_conn.commit() log.info(cursor._executed)
def mysql_failover(master, dry_run, skip_lock, ignore_dr_slave, trust_me_its_dead, kill_old_master): """ Promote a new MySQL master Args: master - Hostaddr object of the master instance to be demoted dry_run - Do not change state, just do sanity testing and exit skip_lock - Do not take a promotion lock ignore_dr_slave - Ignore the existance of a dr_slave trust_me_its_dead - Do not test to see if the master is dead kill_old_master - Send a mysqladmin kill command to the old master Returns: new_master - The new master server """ log.info('Master to demote is {master}'.format(master=master)) zk = host_utils.MysqlZookeeper() (replica_set, _) = zk.get_replica_set_from_instance(master, rtypes=['master']) log.info('Replica set is detected as ' '{replica_set}'.format(replica_set=replica_set)) # take a lock here to make sure nothing changes underneath us if not skip_lock and not dry_run: log.info('Taking promotion lock on replica set') lock_identifier = get_promotion_lock(replica_set) else: lock_identifier = None # giant try. If there any problems we roll back from the except try: master_conn = False slave = zk.get_mysql_instance_from_replica_set( replica_set=replica_set, repl_type=host_utils.REPLICA_ROLE_SLAVE) log.info('Slave/new master is detected as {slave}'.format(slave=slave)) if ignore_dr_slave: log.info('Intentionally ignoring a dr_slave') dr_slave = None else: dr_slave = zk.get_mysql_instance_from_replica_set( replica_set, host_utils.REPLICA_ROLE_DR_SLAVE) log.info( 'DR slave is detected as {dr_slave}'.format(dr_slave=dr_slave)) if dr_slave: if dr_slave == slave: raise Exception('Slave and dr_slave appear to be the same') replicas = set([slave, dr_slave]) else: replicas = set([slave]) # We use master_conn as a mysql connection to the master server, if # it is False, the master is dead if trust_me_its_dead: master_conn = None else: master_conn = is_master_alive(master, replicas) # Test to see if the slave is setup for replication. If not, we are hosed log.info('Testing to see if Slave/new master is setup to write ' 'replication logs') mysql_lib.get_master_status(slave) if kill_old_master and not dry_run: log.info('Killing old master, we hope you know what you are doing') mysql_lib.shutdown_mysql(master) master_conn = None if master_conn: log.info('Master is considered alive') dead_master = False confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_NORMAL, dead_master) else: log.info('Master is considered dead') dead_master = True confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_LOOSE, dead_master) if dry_run: log.info('In dry_run mode, so exiting now') # Using os._exit in order to not get catch in the giant try os._exit(environment_specific.DRY_RUN_EXIT_CODE) log.info('Preliminary sanity checks complete, starting promotion') if master_conn: log.info('Setting read_only on master') mysql_lib.set_global_variable(master, 'read_only', True) log.info('Confirming no writes to old master') # If there are writes with the master in read_only mode then the # promotion can not proceed. # A likely reason is a client has the SUPER privilege. confirm_no_writes(master) log.info('Waiting for replicas to be caught up') confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_NONE, dead_master, True, mysql_lib.NORMAL_HEARTBEAT_LAG) log.info('Setting up replication from old master ({master}) ' 'to new master ({slave})'.format(master=master, slave=slave)) mysql_lib.setup_replication(new_master=slave, new_replica=master) else: log.info('Starting up a zk connection to make sure we can connect') kazoo_client = environment_specific.get_kazoo_client() if not kazoo_client: raise Exception('Could not conect to zk') log.info('Confirming replica has processed all replication ' ' logs') confirm_no_writes(slave) log.info('Looks like no writes being processed by replica via ' 'replication or other means') if len(replicas) > 1: log.info('Confirming replica servers are synced') confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_LOOSE, dead_master, True) except: log.info('Starting rollback') if master_conn: log.info('Releasing read_only on old master') mysql_lib.set_global_variable(master, 'read_only', False) log.info('Clearing replication settings on old master') mysql_lib.reset_slave(master) if lock_identifier: log.info('Releasing promotion lock') release_promotion_lock(lock_identifier) log.info('Rollback complete, reraising exception') raise if dr_slave: try: mysql_lib.setup_replication(new_master=slave, new_replica=dr_slave) except Exception as e: log.error(e) log.error('Setting up replication on the dr_slave failed. ' 'Failing forward!') log.info('Updating zk') zk_write_attempt = 0 while True: try: modify_mysql_zk.swap_master_and_slave(slave, dry_run=False) break except: if zk_write_attempt > MAX_ZK_WRITE_ATTEMPTS: log.info('Final failure writing to zk, bailing') raise else: log.info('Write to zk failed, trying again') zk_write_attempt = zk_write_attempt + 1 log.info('Removing read_only from new master') mysql_lib.set_global_variable(slave, 'read_only', False) log.info('Removing replication configuration from new master') mysql_lib.reset_slave(slave) if lock_identifier: log.info('Releasing promotion lock') release_promotion_lock(lock_identifier) log.info('Failover complete') # we don't really care if this fails, but we'll print a message anyway. try: environment_specific.generic_json_post( environment_specific.CHANGE_FEED_URL, { 'type': 'MySQL Failover', 'environment': replica_set, 'description': "Failover from {m} to {s}".format(m=master, s=slave), 'author': host_utils.get_user(), 'automation': False, 'source': "mysql_failover.py on {}".format(host_utils.HOSTNAME) }) except Exception as e: log.warning("Failover completed, but change feed " "not updated: {}".format(e)) if not master_conn: log.info('As master is dead, will try to launch a replacement. Will ' 'sleep 20 seconds first to let things settle') time.sleep(20) launch_replacement_db_host.launch_replacement_db_host(master)
def mysql_failover(master, dry_run, skip_lock, ignore_dr_slave, trust_me_its_dead, kill_old_master): """ Promote a new MySQL master Args: master - Hostaddr object of the master instance to be demoted dry_run - Do not change state, just do sanity testing and exit skip_lock - Do not take a promotion lock ignore_dr_slave - Ignore the existance of a dr_slave trust_me_its_dead - Do not test to see if the master is dead kill_old_master - Send a mysqladmin kill command to the old master Returns: new_master - The new master server """ log.info('Master to demote is {master}'.format(master=master)) zk = host_utils.MysqlZookeeper() (replica_set, _) = zk.get_replica_set_from_instance(master, rtypes=['master']) log.info('Replica set is detected as ' '{replica_set}'.format(replica_set=replica_set)) # take a lock here to make sure nothing changes underneath us if not skip_lock and not dry_run: log.info('Taking promotion lock on replica set') lock_identifier = get_promotion_lock(replica_set) else: lock_identifier = None # giant try. If there any problems we roll back from the except try: master_conn = False slave = zk.get_mysql_instance_from_replica_set(replica_set=replica_set, repl_type=host_utils.REPLICA_ROLE_SLAVE) log.info('Slave/new master is detected as {slave}'.format(slave=slave)) if ignore_dr_slave: log.info('Intentionally ignoring a dr_slave') dr_slave = None else: dr_slave = zk.get_mysql_instance_from_replica_set(replica_set, host_utils.REPLICA_ROLE_DR_SLAVE) log.info('DR slave is detected as {dr_slave}'.format(dr_slave=dr_slave)) if dr_slave: if dr_slave == slave: raise Exception('Slave and dr_slave appear to be the same') replicas = set([slave, dr_slave]) else: replicas = set([slave]) # We use master_conn as a mysql connection to the master server, if # it is False, the master is dead if trust_me_its_dead: master_conn = None else: master_conn = is_master_alive(master, replicas) # Test to see if the slave is setup for replication. If not, we are hosed log.info('Testing to see if Slave/new master is setup to write ' 'replication logs') mysql_lib.get_master_status(slave) if kill_old_master and not dry_run: log.info('Killing old master, we hope you know what you are doing') mysql_lib.shutdown_mysql(master) master_conn = None if master_conn: log.info('Master is considered alive') dead_master = False confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_NORMAL, dead_master) else: log.info('Master is considered dead') dead_master = True confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_LOOSE, dead_master) if dry_run: log.info('In dry_run mode, so exiting now') # Using os._exit in order to not get catch in the giant try os._exit(0) log.info('Preliminary sanity checks complete, starting promotion') if master_conn: log.info('Setting read_only on master') mysql_lib.set_global_variable(master, 'read_only', True) log.info('Confirming no writes to old master') # If there are writes with the master in read_only mode then the # promotion can not proceed. # A likely reason is a client has the SUPER privilege. confirm_no_writes(master) log.info('Waiting for replicas to be caught up') confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_NONE, dead_master, True, mysql_lib.NORMAL_HEARTBEAT_LAG) log.info('Setting up replication from old master ({master}) ' 'to new master ({slave})'.format(master=master, slave=slave)) mysql_lib.setup_replication(new_master=slave, new_replica=master) else: log.info('Starting up a zk connection to make sure we can connect') kazoo_client = environment_specific.get_kazoo_client() if not kazoo_client: raise Exception('Could not conect to zk') log.info('Confirming replica has processed all replication ' ' logs') confirm_no_writes(slave) log.info('Looks like no writes being processed by replica via ' 'replication or other means') if len(replicas) > 1: log.info('Confirming replica servers are synced') confirm_max_replica_lag(replicas, mysql_lib.REPLICATION_TOLERANCE_LOOSE, dead_master, True) except: log.info('Starting rollback') if master_conn: log.info('Releasing read_only on old master') mysql_lib.set_global_variable(master, 'read_only', False) log.info('Clearing replication settings on old master') mysql_lib.reset_slave(master) if lock_identifier: log.info('Releasing promotion lock') release_promotion_lock(lock_identifier) log.info('Rollback complete, reraising exception') raise if dr_slave: try: mysql_lib.setup_replication(new_master=slave, new_replica=dr_slave) except Exception as e: log.error(e) log.error('Setting up replication on the dr_slave failed. ' 'Failing forward!') log.info('Updating zk') zk_write_attempt = 0 while True: try: modify_mysql_zk.swap_master_and_slave(slave, dry_run=False) break except: if zk_write_attempt > MAX_ZK_WRITE_ATTEMPTS: log.info('Final failure writing to zk, bailing') raise else: log.info('Write to zk failed, trying again') zk_write_attempt = zk_write_attempt+1 log.info('Removing read_only from new master') mysql_lib.set_global_variable(slave, 'read_only', False) log.info('Removing replication configuration from new master') mysql_lib.reset_slave(slave) if lock_identifier: log.info('Releasing promotion lock') release_promotion_lock(lock_identifier) log.info('Failover complete') # we don't really care if this fails, but we'll print a message anyway. try: environment_specific.generic_json_post( environment_specific.CHANGE_FEED_URL, {'type': 'MySQL Failover', 'environment': replica_set, 'description': "Failover from {m} to {s}".format(m=master, s=slave), 'author': host_utils.get_user(), 'automation': False, 'source': "mysql_failover.py on {}".format(host_utils.HOSTNAME)}) except Exception as e: log.warning("Failover completed, but change feed " "not updated: {}".format(e)) if not master_conn: log.info('As master is dead, will try to launch a replacement. Will ' 'sleep 20 seconds first to let things settle') time.sleep(20) launch_replacement_db_host.launch_replacement_db_host(master)