def purge_mysql_backups(instance, skip_lock=False):
    """ Run a file based backup on a supplied local instance

    Args:
    instance - A hostaddr object
    skip_lock - Don't take out a lock against other backup related functions
                running
    """
    lock_handle = None
    try:
        if not skip_lock:
            log.info('Taking backup lock')
            lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)

        (temp_path, target_path) = backup.get_paths(str(instance.port))

        log.info("Cleaning up any partial backups")
        backup.remove_backups(temp_path,
                              keep_newest=0,
                              extension=('.xbstream', '.sql.gz'))

        log.info("Purge xtrabackup backups to the "
                 "last {cnt}".format(cnt=KEEP_OLD_XTRABACKUP))
        backup.remove_backups(target_path,
                              keep_newest=KEEP_OLD_XTRABACKUP,
                              extension=('.xbstream'))

        log.info("Purge logical backup backups to the "
                 "last {cnt}".format(cnt=KEEP_OLD_LOGICAL))
        backup.remove_backups(target_path,
                              keep_newest=KEEP_OLD_LOGICAL,
                              extension=('.sql.gz'))

        log.info("Chmod'ing {target}".format(target=backup.TARGET_DIR))
        host_utils.change_perms(backup.TARGET_DIR, 777)

        log.info("Chmod'ing {temp}".format(temp=backup.TEMP_DIR))
        host_utils.change_perms(backup.TEMP_DIR, 777)

    finally:
        if not skip_lock and lock_handle:
            log.info('Releasing lock')
            host_utils.release_flock_lock(lock_handle)
def purge_mysql_backups(instance, skip_lock=False):
    """ Run a file based backup on a supplied local instance

    Args:
    instance - A hostaddr object
    skip_lock - Don't take out a lock against other backup related functions
                running
    """
    lock_handle = None
    try:
        if not skip_lock:
            log.info('Taking backup lock')
            lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)

        (temp_path, target_path) = backup.get_paths(str(instance.port))

        log.info("Cleaning up any partial backups")
        backup.remove_backups(temp_path,
                              keep_newest=0,
                              extension=('.xbstream', '.sql.gz'))

        log.info("Purge xtrabackup backups to the "
                 "last {cnt}".format(cnt=KEEP_OLD_XTRABACKUP))
        backup.remove_backups(target_path,
                              keep_newest=KEEP_OLD_XTRABACKUP,
                              extension=('.xbstream'))

        log.info("Purge logical backup backups to the "
                 "last {cnt}".format(cnt=KEEP_OLD_LOGICAL))
        backup.remove_backups(target_path,
                              keep_newest=KEEP_OLD_LOGICAL,
                              extension=('.sql.gz'))

        log.info("Chmod'ing {target}".format(target=backup.TARGET_DIR))
        host_utils.change_perms(backup.TARGET_DIR, 777)

        log.info("Chmod'ing {temp}".format(temp=backup.TEMP_DIR))
        host_utils.change_perms(backup.TEMP_DIR, 777)

    finally:
        if not skip_lock and lock_handle:
            log.info('Releasing lock')
            host_utils.release_flock_lock(lock_handle)
def find_a_backup_to_restore(restore_type, source, destination, date):
    """ Based on supplied constains, try to find a backup to restore

    Args:
    restore_type - What possible methods to use to pull the backup,
                   options are 's3', 'remote_server'.
    source - A hostaddr object for where to pull a backup from
    destination -  A hostaddr object for where to restore the backup
    date - What date should the backup be from

    Returns:
    restore_type - Which method to download a backup shoudl be used
    restore_source - Where the backup was taken
    retore_file - Where the file exists on whichever storage
    restore_size - What is the size of the backup in bytes
    """
    zk = host_utils.MysqlZookeeper()
    possible_sources = set()
    if source:
        # the souce may not be in zk because it is a new replica set
        possible_sources.add(source)
        if source.get_zk_replica_set():
            replica_set = source.get_zk_replica_set()[0]
        else:
            replica_set = None
    else:
        replica_set = destination.get_zk_replica_set()[0]
        for role in host_utils.REPLICA_TYPES:
            possible_sources.add(zk.get_mysql_instance_from_replica_set(replica_set, role))
    log.info('Replica set detected as {replica_set}'.format(replica_set=replica_set))
    log.info('Possible source hosts:{possible_sources}'.format(possible_sources=possible_sources))

    if date:
        dates = [date]
    else:
        dates = []
        for days in range(0, DEFAULT_MAX_RESTORE_AGE):
            dates.append(datetime.date.today() - datetime.timedelta(days=days))

    # Find a backup file, preferably newer and less strong preferece on the master server
    restore_file = None
    for restore_date in dates:
        if restore_file:
            break

        log.info('Looking for a backup for {restore_date}'.format(restore_date=restore_date))
        if not restore_type or restore_type == 's3':
            log.info('Looking for a backup in s3')
            for possible_source in possible_sources:
                try:
                    (restore_file, restore_size) = backup.get_s3_backup(possible_source, str(restore_date))
                    restore_source = possible_source
                    restore_type = 's3'
                    break
                except:
                    log.info('No backup found on in s3 for {source}'.format(source=possible_source))
            if not restore_file:
                log.info('Could not find a backup in S3')

        # Perhaps the local filesystem of a remote servers?
        if not restore_type or restore_type == 'remote_server':
            log.info('Looking for a backup on remote filesystems')
            for possible_source in possible_sources:
                try:
                    (restore_file, restore_size) = backup.get_remote_backup(possible_source, str(restore_date))
                    restore_source = possible_source
                    restore_type = 'remote_server'
                    break
                except:
                    log.info('No backup found on remote filesystem for {source}'.format(source=possible_source))
            if not restore_file:
                log.info('Could not find a backup on remote filesystems')

    if not restore_file:
        raise Exception('Could not find a backup to restore')

    log.info('Found a backup {restore_file} via '
             '{restore_type}'.format(restore_file=restore_file,
                                     restore_type=restore_type))
    (temp_path, target_path) = backup.get_paths(destination.port)

    return restore_type, restore_source, restore_file, restore_size
def restore_instance(restore_source, destination, restore_type,
                     restore_file, no_repl, date,
                     add_to_zk, skip_production_check,
                     test_restore):
    """ Restore a MySQL backup on to localhost

    Args:
    restore_source - A hostaddr object for where to pull a backup from
    destination -  A hostaddr object for where to restore the backup
    restore_type - How to pull the backup, options are 's3', 'remote_server'
                   and 'local_file'
    no_repl - Should  replication be not started. It will always be setup.
    date - What date should the backup be from
    add_to_zk - Should the instnace be added to zk. If so, the log from the
                host being launched will be consulted.
    skip_production_check - Do not check if the host is already in zk for
                            production use.
    test_restore - Use less ram and shutdown the instance after going
                   through the motions of a restore.
    """
    (temp_dir, target_dir) = backup.get_paths(str(destination.port))
    log.info('Supplied source is {source}'.format(source=restore_source))
    log.info('Supplied destination is {dest}'.format(dest=destination))
    log.info('Restore type is {rest}'.format(rest=restore_type))
    log.info('Local restore file is {file}'.format(file=restore_file))
    log.info('Desired date of restore {date}'.format(date=date))
    if test_restore == 'test':
        log.info('Running restore in test mode')

    # Try to prevent unintentional destruction of prod servers
    zk = host_utils.MysqlZookeeper()
    try:
        (_, replica_type) = zk.get_replica_set_from_instance(destination)
    except:
        # instance is not in production
        replica_type = None
    if replica_type == host_utils.REPLICA_ROLE_MASTER:
        # If the instance, we will refuse to run. No ifs, ands, or buts/
        raise Exception('Restore script must never run on a master')
    if replica_type:
        if skip_production_check:
            log.info('Ignoring production check. We hope you know what you '
                     'are doing and we will try to take a backup in case '
                     'you are wrong.')
            try:
                mysql_backup.mysql_backup(destination)
            except Exception as e:
                log.error(e)
                log.warning('Unable to take a backup. We will give you {time} '
                            'seconds to change your mind and ^c.'
                            ''.format(time=SCARY_TIMEOUT))
                time.sleep(SCARY_TIMEOUT)
        else:
            raise Exception("It appears {instance} is in use. This is"
                            " very dangerous!".format(instance=destination))

    # Take a lock to prevent multiple restores from running concurrently
    log.info('Taking a flock to block another restore from starting')
    lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)

    log.info('Rebuilding cnf files just in case')
    mysql_cnf_builder.build_cnf()

    mysql_init_server.create_and_chown_dirs(destination.port)

    # load some data from the mysql conf file
    datadir = host_utils.get_cnf_setting('datadir', destination.port)

    # Where will we look for a backup?
    if restore_type != 'local_file':
        (restore_type, restore_source,
         restore_file, restore_size) = find_a_backup_to_restore(restore_type, restore_source,
                                                                destination, date)
    # Not using an if/else because find_a_backup_to_restore could set to
    # local_file if the file has already been downloaded.
    if restore_type == 'local_file':
        restore_source = backup.get_host_from_backup(restore_file)
        # restore_size will be computed in the unpack function
        restore_size = None
        log.info('Detected the source of backup as {src}'.format(src=restore_source))

    if restore_source.get_zk_replica_set():
        replica_set = restore_source.get_zk_replica_set()[0]
        master = zk.get_mysql_instance_from_replica_set(replica_set, host_utils.REPLICA_ROLE_MASTER)
    else:
        # ZK has no idea what this replica set is, probably a new replica set.
        master = restore_source

    # Start logging
    row_id = backup.start_restore_log(master, {'restore_type': restore_type,
                                               'test_restore': test_restore,
                                               'restore_source': restore_source,
                                               'restore_port': destination.port,
                                               'restore_file': restore_file,
                                               'source_instance': destination.hostname,
                                               'restore_date': date,
                                               'replication': no_repl,
                                               'zookeeper': add_to_zk})
    # Giant try to allow logging if anything goes wrong.
    try:
        # If we hit an exception, this status will be used. If not, it will
        # be overwritten
        restore_log_update = {'restore_status': 'BAD'}
        log.info('Quick sanity check')
        mysql_init_server.basic_host_sanity()

        log.info('Shutting down MySQL')
        host_utils.stop_mysql(destination.port)

        log.info('Removing any existing MySQL data')
        mysql_init_server.delete_mysql_data(destination.port)

        log.info('Unpacking {rfile} into {ddir}'.format(rfile=restore_file,
                                                        ddir=datadir))
        backup.xbstream_unpack(restore_file, destination.port,
                               restore_source, restore_type, restore_size)

        log.info('Decompressing files in {path}'.format(path=datadir))
        backup.innobackup_decompress(destination.port)

        log.info('Applying logs')
        if test_restore == 'test':
            # We don't really need a lot of memory if we're just
            # verifying that it works.
            backup.apply_log(destination.port, memory='1G')
        else:
            backup.apply_log(destination.port, memory='10G')

        log.info('Removing old innodb redo logs')
        mysql_init_server.delete_innodb_log_files(destination.port)

        log.info('Setting permissions for MySQL on {dir}'.format(dir=datadir))
        host_utils.change_owner(datadir, 'mysql', 'mysql')

        log.info('Starting MySQL')
        host_utils.upgrade_auth_tables(destination.port)
        restore_log_update = {'restore_status': 'OK'}

        log.info('Running MySQL upgrade')
        host_utils.start_mysql(destination.port,
                               options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))

        if master == backup.get_host_from_backup(restore_file):
            log.info('Pulling replication info from restore to backup source')
            (binlog_file, binlog_pos) = backup.parse_xtrabackup_binlog_info(datadir)
        else:
            log.info('Pulling replication info from restore to '
                     'master of backup source')
            (binlog_file, binlog_pos) = backup.parse_xtrabackup_slave_info(datadir)

        log.info('Setting up MySQL replication')
        restore_log_update['replication'] = 'FAIL'

        # Since we haven't started the slave yet, make sure we've got these
        # plugins installed, whether we use them or not.
        mysql_lib.setup_semisync_plugins(destination)

        # Try to configure replication.  If this was just a test restore,
        # don't wait for it to catch up - don't even start the slave.
        if test_restore == 'test':
            mysql_lib.change_master(destination,
                                    master,
                                    binlog_file,
                                    binlog_pos,
                                    no_start=True)
            backup.quick_test_replication(destination)
        else:
            mysql_lib.change_master(destination,
                                    master,
                                    binlog_file,
                                    binlog_pos,
                                    no_start=(no_repl == 'SKIP'))
            mysql_lib.wait_replication_catch_up(destination)
            host_utils.restart_pt_daemons(destination.port)

        restore_log_update['replication'] = 'OK'

        mysql_lib.setup_response_time_metrics(destination)

    except Exception as e:
        log.error(e)
        if row_id is not None:
            restore_log_update['status_message'] = e
            restore_log_update['finished_at'] = True
        raise
    finally:
        if lock_handle:
            log.info('Releasing lock')
            host_utils.release_flock_lock(lock_handle)
        backup.update_restore_log(master, row_id, restore_log_update)

    # If this was a test restore, we don't need to keep the 3307
    # instance running, so let's shut it off.
    if test_restore == 'test':
        log.info('Shutting down MySQL backup/restore test instance')
        host_utils.stop_mysql(destination.port)
        backup.update_restore_log(master, row_id, {'finished_at': True})
        return

    try:
        if add_to_zk == 'REQ':
            log.info('Adding instance to zk')
            modify_mysql_zk.auto_add_instance_to_zk(destination, dry_run=False)
            backup.update_restore_log(master, row_id, {'zookeeper': 'OK'})
        else:
            log.info('add_to_zk is not set, therefore not adding to zk')
    except Exception as e:
        log.warning("An exception occurred: {e}".format(e=e))
        log.warning("If this is a DB issue, that's fine. "
                    "Otherwise, you should check ZK.")

    backup.update_restore_log(master, row_id, {'finished_at': True})
    log.info('Starting a new backup')
    mysql_backup.mysql_backup(destination)
Beispiel #5
0
def find_a_backup_to_restore(restore_type, source, destination, date):
    """ Based on supplied constains, try to find a backup to restore

    Args:
    restore_type - What possible methods to use to pull the backup,
                   options are 's3', 'remote_server'.
    source - A hostaddr object for where to pull a backup from
    destination -  A hostaddr object for where to restore the backup
    date - What date should the backup be from

    Returns:
    restore_type - Which method to download a backup shoudl be used
    restore_source - Where the backup was taken
    retore_file - Where the file exists on whichever storage
    restore_size - What is the size of the backup in bytes
    """
    zk = host_utils.MysqlZookeeper()
    possible_sources = set()
    if source:
        # the souce may not be in zk because it is a new replica set
        possible_sources.add(source)
        if source.get_zk_replica_set():
            replica_set = source.get_zk_replica_set()[0]
        else:
            replica_set = None
    else:
        replica_set = destination.get_zk_replica_set()[0]
        for role in host_utils.REPLICA_TYPES:
            possible_sources.add(
                zk.get_mysql_instance_from_replica_set(replica_set, role))
    log.info('Replica set detected as {replica_set}'.format(
        replica_set=replica_set))
    log.info('Possible source hosts:{possible_sources}'.format(
        possible_sources=possible_sources))

    if date:
        dates = [date]
    else:
        dates = []
        for days in range(0, DEFAULT_MAX_RESTORE_AGE):
            dates.append(datetime.date.today() - datetime.timedelta(days=days))

    # Find a backup file, preferably newer and less strong preferece on the master server
    restore_file = None
    for restore_date in dates:
        if restore_file:
            break

        log.info('Looking for a backup for {restore_date}'.format(
            restore_date=restore_date))
        if not restore_type or restore_type == 's3':
            log.info('Looking for a backup in s3')
            for possible_source in possible_sources:
                try:
                    (restore_file, restore_size) = backup.get_s3_backup(
                        possible_source, str(restore_date))
                    restore_source = possible_source
                    restore_type = 's3'
                    break
                except:
                    log.info('No backup found on in s3 for {source}'.format(
                        source=possible_source))
            if not restore_file:
                log.info('Could not find a backup in S3')

        # Perhaps the local filesystem of a remote servers?
        if not restore_type or restore_type == 'remote_server':
            log.info('Looking for a backup on remote filesystems')
            for possible_source in possible_sources:
                try:
                    (restore_file, restore_size) = backup.get_remote_backup(
                        possible_source, str(restore_date))
                    restore_source = possible_source
                    restore_type = 'remote_server'
                    break
                except:
                    log.info(
                        'No backup found on remote filesystem for {source}'.
                        format(source=possible_source))
            if not restore_file:
                log.info('Could not find a backup on remote filesystems')

    if not restore_file:
        raise Exception('Could not find a backup to restore')

    log.info('Found a backup {restore_file} via '
             '{restore_type}'.format(restore_file=restore_file,
                                     restore_type=restore_type))
    (temp_path, target_path) = backup.get_paths(destination.port)

    return restore_type, restore_source, restore_file, restore_size
Beispiel #6
0
def restore_instance(restore_source, destination, restore_type, restore_file,
                     no_repl, date, add_to_zk, skip_production_check,
                     test_restore):
    """ Restore a MySQL backup on to localhost

    Args:
    restore_source - A hostaddr object for where to pull a backup from
    destination -  A hostaddr object for where to restore the backup
    restore_type - How to pull the backup, options are 's3', 'remote_server'
                   and 'local_file'
    no_repl - Should  replication be not started. It will always be setup.
    date - What date should the backup be from
    add_to_zk - Should the instnace be added to zk. If so, the log from the
                host being launched will be consulted.
    skip_production_check - Do not check if the host is already in zk for
                            production use.
    test_restore - Use less ram and shutdown the instance after going
                   through the motions of a restore.
    """
    (temp_dir, target_dir) = backup.get_paths(str(destination.port))
    log.info('Supplied source is {source}'.format(source=restore_source))
    log.info('Supplied destination is {dest}'.format(dest=destination))
    log.info('Restore type is {rest}'.format(rest=restore_type))
    log.info('Local restore file is {file}'.format(file=restore_file))
    log.info('Desired date of restore {date}'.format(date=date))
    if test_restore == 'test':
        log.info('Running restore in test mode')

    # Try to prevent unintentional destruction of prod servers
    zk = host_utils.MysqlZookeeper()
    try:
        (_, replica_type) = zk.get_replica_set_from_instance(destination)
    except:
        # instance is not in production
        replica_type = None
    if replica_type == host_utils.REPLICA_ROLE_MASTER:
        # If the instance, we will refuse to run. No ifs, ands, or buts/
        raise Exception('Restore script must never run on a master')
    if replica_type:
        if skip_production_check:
            log.info('Ignoring production check. We hope you know what you '
                     'are doing and we will try to take a backup in case '
                     'you are wrong.')
            try:
                mysql_backup.mysql_backup(destination)
            except Exception as e:
                log.error(e)
                log.warning('Unable to take a backup. We will give you {time} '
                            'seconds to change your mind and ^c.'
                            ''.format(time=SCARY_TIMEOUT))
                time.sleep(SCARY_TIMEOUT)
        else:
            raise Exception("It appears {instance} is in use. This is"
                            " very dangerous!".format(instance=destination))

    # Take a lock to prevent multiple restores from running concurrently
    log.info('Taking a flock to block another restore from starting')
    lock_handle = host_utils.take_flock_lock(backup.BACKUP_LOCK_FILE)

    log.info('Rebuilding cnf files just in case')
    mysql_cnf_builder.build_cnf()

    mysql_init_server.create_and_chown_dirs(destination.port)

    # load some data from the mysql conf file
    datadir = host_utils.get_cnf_setting('datadir', destination.port)

    # Where will we look for a backup?
    if restore_type != 'local_file':
        (restore_type, restore_source, restore_file,
         restore_size) = find_a_backup_to_restore(restore_type, restore_source,
                                                  destination, date)
    # Not using an if/else because find_a_backup_to_restore could set to
    # local_file if the file has already been downloaded.
    if restore_type == 'local_file':
        restore_source = backup.get_host_from_backup(restore_file)
        # restore_size will be computed in the unpack function
        restore_size = None
        log.info('Detected the source of backup as {src}'.format(
            src=restore_source))

    if restore_source.get_zk_replica_set():
        replica_set = restore_source.get_zk_replica_set()[0]
        master = zk.get_mysql_instance_from_replica_set(
            replica_set, host_utils.REPLICA_ROLE_MASTER)
    else:
        # ZK has no idea what this replica set is, probably a new replica set.
        master = restore_source

    # Start logging
    row_id = backup.start_restore_log(
        master, {
            'restore_type': restore_type,
            'test_restore': test_restore,
            'restore_source': restore_source,
            'restore_port': destination.port,
            'restore_file': restore_file,
            'source_instance': destination.hostname,
            'restore_date': date,
            'replication': no_repl,
            'zookeeper': add_to_zk
        })
    # Giant try to allow logging if anything goes wrong.
    try:
        # If we hit an exception, this status will be used. If not, it will
        # be overwritten
        restore_log_update = {'restore_status': 'BAD'}
        log.info('Quick sanity check')
        mysql_init_server.basic_host_sanity()

        log.info('Shutting down MySQL')
        host_utils.stop_mysql(destination.port)

        log.info('Removing any existing MySQL data')
        mysql_init_server.delete_mysql_data(destination.port)

        log.info('Unpacking {rfile} into {ddir}'.format(rfile=restore_file,
                                                        ddir=datadir))
        backup.xbstream_unpack(restore_file, destination.port, restore_source,
                               restore_type, restore_size)

        log.info('Decompressing files in {path}'.format(path=datadir))
        backup.innobackup_decompress(destination.port)

        log.info('Applying logs')
        if test_restore == 'test':
            # We don't really need a lot of memory if we're just
            # verifying that it works.
            backup.apply_log(destination.port, memory='1G')
        else:
            backup.apply_log(destination.port, memory='10G')

        log.info('Removing old innodb redo logs')
        mysql_init_server.delete_innodb_log_files(destination.port)

        log.info('Setting permissions for MySQL on {dir}'.format(dir=datadir))
        host_utils.change_owner(datadir, 'mysql', 'mysql')

        log.info('Starting MySQL')
        host_utils.upgrade_auth_tables(destination.port)
        restore_log_update = {'restore_status': 'OK'}

        log.info('Running MySQL upgrade')
        host_utils.start_mysql(
            destination.port,
            options=host_utils.DEFAULTS_FILE_EXTRA_ARG.format(
                defaults_file=host_utils.MYSQL_NOREPL_CNF_FILE))

        if master == backup.get_host_from_backup(restore_file):
            log.info('Pulling replication info from restore to backup source')
            (binlog_file,
             binlog_pos) = backup.parse_xtrabackup_binlog_info(datadir)
        else:
            log.info('Pulling replication info from restore to '
                     'master of backup source')
            (binlog_file,
             binlog_pos) = backup.parse_xtrabackup_slave_info(datadir)

        log.info('Setting up MySQL replication')
        restore_log_update['replication'] = 'FAIL'

        # Since we haven't started the slave yet, make sure we've got these
        # plugins installed, whether we use them or not.
        mysql_lib.setup_semisync_plugins(destination)

        # Try to configure replication.  If this was just a test restore,
        # don't wait for it to catch up - don't even start the slave.
        if test_restore == 'test':
            mysql_lib.change_master(destination,
                                    master,
                                    binlog_file,
                                    binlog_pos,
                                    no_start=True)
            backup.quick_test_replication(destination)
        else:
            mysql_lib.change_master(destination,
                                    master,
                                    binlog_file,
                                    binlog_pos,
                                    no_start=(no_repl == 'SKIP'))
            mysql_lib.wait_replication_catch_up(destination)
            host_utils.restart_pt_daemons(destination.port)

        restore_log_update['replication'] = 'OK'

        mysql_lib.setup_response_time_metrics(destination)

    except Exception as e:
        log.error(e)
        if row_id is not None:
            restore_log_update['status_message'] = e
            restore_log_update['finished_at'] = True
        raise
    finally:
        if lock_handle:
            log.info('Releasing lock')
            host_utils.release_flock_lock(lock_handle)
        backup.update_restore_log(master, row_id, restore_log_update)

    # If this was a test restore, we don't need to keep the 3307
    # instance running, so let's shut it off.
    if test_restore == 'test':
        log.info('Shutting down MySQL backup/restore test instance')
        host_utils.stop_mysql(destination.port)
        backup.update_restore_log(master, row_id, {'finished_at': True})
        return

    try:
        if add_to_zk == 'REQ':
            log.info('Adding instance to zk')
            modify_mysql_zk.auto_add_instance_to_zk(destination, dry_run=False)
            backup.update_restore_log(master, row_id, {'zookeeper': 'OK'})
        else:
            log.info('add_to_zk is not set, therefore not adding to zk')
    except Exception as e:
        log.warning("An exception occurred: {e}".format(e=e))
        log.warning("If this is a DB issue, that's fine. "
                    "Otherwise, you should check ZK.")

    backup.update_restore_log(master, row_id, {'finished_at': True})
    log.info('Starting a new backup')
    mysql_backup.mysql_backup(destination)