Beispiel #1
0
    def backup(self):
        """Run a MySQL backup"""

        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        mock_env = None
        if self.dry_run:
            mock_env = MockEnvironment()
            mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")

        try:
            if self.config["mysqldump"]["stop-slave"]:
                self.client = connect(self.mysql_config["client"])
                if self.client.show_status("Slave_running", session=None) != "ON":
                    raise BackupError(
                        "stop-slave enabled, but replication is "
                        "either not configured or the slave is not "
                        "running."
                    )
                self.config.setdefault("mysql:replication", {})
                _stop_slave(self.client, self.config["mysql:replication"])
            self._backup()
        finally:
            if self.config["mysqldump"]["stop-slave"] and "mysql:replication" in self.config:
                _start_slave(self.client, self.config["mysql:replication"])
            if mock_env:
                mock_env.restore_environment()
Beispiel #2
0
    def backup(self):
        """Run a MySQL backup"""

        mock_env = None
        if self.dry_run:
            mock_env = MockEnvironment()
            mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")

        try:
            self._backup()
        finally:
            if mock_env:
                mock_env.restore_environment()
Beispiel #3
0
    def backup(self):
        """Run a MySQL backup"""
        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        try:
            self.client = connect(self.mysql_config["client"])
        except Exception as ex:
            LOG.debug("%s", ex)
            raise BackupError("Failed connecting to database'")

        if self.dry_run:
            self.mock_env = MockEnvironment()
            self.mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")
            status = self.client.show_databases()
            if not status:
                raise BackupError("Failed to run 'show databases'")
        try:
            if self.config["mysqldump"]["stop-slave"]:
                slave_status = self.client.show_slave_status()
                if not slave_status:
                    raise BackupError("stop-slave enabled, but 'show slave "
                                      "status' failed")
                if slave_status and slave_status["slave_sql_running"] != "Yes":
                    raise BackupError("stop-slave enabled, but replication is "
                                      "not running")
                if not self.dry_run:
                    _stop_slave(self.client, self.config)
            elif self.config["mysqldump"]["bin-log-position"]:
                self.config["mysql:replication"] = {}
                repl_cfg = self.config["mysql:replication"]
                try:
                    master_info = self.client.show_master_status()
                    if master_info:
                        repl_cfg["master_log_file"] = master_info["file"]
                        repl_cfg["master_log_pos"] = master_info["position"]
                except MySQLError as exc:
                    raise BackupError(
                        "Failed to acquire master status [%d] %s" % exc.args)
            self._backup()
        finally:
            if self.config["mysqldump"][
                    "stop-slave"] and "mysql:replication" in self.config:
                _start_slave(self.client, self.config["mysql:replication"])
            if self.mock_env:
                self.mock_env.restore_environment()
Beispiel #4
0
    def backup(self):
        """Run a MySQL backup"""

        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        mock_env = None
        if self.dry_run:
            mock_env = MockEnvironment()
            mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")

        try:
            if self.config['mysqldump']['stop-slave']:
                self.client = connect(self.mysql_config['client'])
                if self.client.show_status('Slave_running',
                                           session=None) != 'ON':
                    raise BackupError(
                        "stop-slave enabled, but replication is "
                        "either not configured or the slave is not "
                        "running.")
                self.config.setdefault('mysql:replication', {})
                _stop_slave(self.client, self.config['mysql:replication'])
            self._backup()
        finally:
            if self.config['mysqldump']['stop-slave'] and \
                'mysql:replication' in self.config:
                _start_slave(self.client, self.config['mysql:replication'])
            if mock_env:
                mock_env.restore_environment()
Beispiel #5
0
    def backup(self):
        """Run a MySQL backup"""
        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        try:
            self.client = connect(self.mysql_config["client"])
        except Exception as ex:
            LOG.debug("%s", ex)
            raise BackupError("Failed connecting to database'")

        if self.dry_run:
            self.mock_env = MockEnvironment()
            self.mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")
            status = self.client.show_databases()
            if not status:
                raise BackupError("Failed to run 'show databases'")
        try:
            if self.config["mysqldump"]["stop-slave"]:
                slave_status = self.client.show_slave_status()
                if not slave_status:
                    raise BackupError("stop-slave enabled, but 'show slave " "status' failed")
                if slave_status and slave_status["slave_sql_running"] != "Yes":
                    raise BackupError("stop-slave enabled, but replication is " "not running")
                if not self.dry_run:
                    _stop_slave(self.client, self.config)
            elif self.config["mysqldump"]["bin-log-position"]:
                self.config["mysql:replication"] = {}
                repl_cfg = self.config["mysql:replication"]
                try:
                    master_info = self.client.show_master_status()
                    if master_info:
                        repl_cfg["master_log_file"] = master_info["file"]
                        repl_cfg["master_log_pos"] = master_info["position"]
                except MySQLError as exc:
                    raise BackupError("Failed to acquire master status [%d] %s" % exc.args)
            self._backup()
        finally:
            if self.config["mysqldump"]["stop-slave"] and "mysql:replication" in self.config:
                _start_slave(self.client, self.config["mysql:replication"])
            if self.mock_env:
                self.mock_env.restore_environment()
Beispiel #6
0
    def backup(self):
        """Run a MySQL backup"""
        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        mock_env = None
        if self.dry_run:
            mock_env = MockEnvironment()
            mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")

        try:
            if self.config['mysqldump']['stop-slave']:
                self.client = connect(self.mysql_config['client'])
                slave_status = self.client.show_slave_status()
                if slave_status is None:
                    raise BackupError("stop-slave enabled, but 'show slave "
                                      "status' failed")
                elif slave_status['slave_sql_running'] != 'Yes':
                    raise BackupError("stop-slave enabled, but replication is "
                                      "not running")
                _stop_slave(self.client, self.config)
            elif self.config['mysqldump']['bin-log-position']:
                self.config['mysql:replication'] = {}
                repl_cfg = self.config['mysql:replication']
                try:
                    master_info = self.client.show_master_status()
                    if master_info:
                        repl_cfg['master_log_file'] = master_info['file']
                        repl_cfg['master_log_pos'] = master_info['position']
                except MySQLError as exc:
                    raise BackupError(
                        "Failed to acquire master status [%d] %s" % exc.args)
            self._backup()
        finally:
            if self.config['mysqldump']['stop-slave'] and \
                'mysql:replication' in self.config:
                _start_slave(self.client, self.config['mysql:replication'])
            if mock_env:
                mock_env.restore_environment()
Beispiel #7
0
class MySQLDumpPlugin(object):
    """MySQLDump Backup Plugin interface for Holland"""

    CONFIGSPEC = CONFIGSPEC

    def __init__(self, name, config, target_directory, dry_run=False):
        self.name = name
        self.config = config
        self.target_directory = target_directory
        self.dry_run = dry_run
        self.config.validate_config(self.CONFIGSPEC)  # -> ValidationError

        # Setup a discovery shell to find schema items
        # This will iterate over items during the estimate
        # or backup phase, which will call schema.refresh()
        self.schema = MySQLSchema()
        config = self.config["mysqldump"]
        self.schema.add_database_filter(include_glob(*config["databases"]))
        self.schema.add_database_filter(
            exclude_glob(*config["exclude-databases"]))

        self.schema.add_table_filter(include_glob_qualified(*config["tables"]))
        self.schema.add_table_filter(
            exclude_glob_qualified(*config["exclude-tables"]))
        self.schema.add_engine_filter(include_glob(*config["engines"]))
        self.schema.add_engine_filter(exclude_glob(*config["exclude-engines"]))

        self.mysql_config = build_mysql_config(self.config["mysql:client"])
        self.client = connect(self.mysql_config["client"])

        self.mock_env = None

    def estimate_backup_size(self):
        """Estimate the size of the backup this plugin will generate"""

        LOG.info("Estimating size of mysqldump backup")
        estimate_method = self.config["mysqldump"]["estimate-method"]

        if estimate_method.startswith("const:"):
            try:
                return parse_size(estimate_method[6:])
            except ValueError as exc:
                raise BackupError(str(exc))

        if estimate_method != "plugin":
            raise BackupError("Invalid estimate-method '%s'" % estimate_method)

        try:
            db_iter = DatabaseIterator(self.client)
            tbl_iter = MetadataTableIterator(self.client)
            try:
                self.client.connect()
            except Exception as ex:
                LOG.error("Failed to connect to database")
                LOG.debug("%s", ex)
                raise BackupError("MySQL Error %s" % ex)
            try:
                self.schema.refresh(db_iter=db_iter, tbl_iter=tbl_iter)
            except MySQLError as exc:
                LOG.error("Failed to estimate backup size")
                LOG.debug("[%d] %s", *exc.args)
                raise BackupError("MySQL Error [%d] %s" % exc.args)
            return float(sum([db.size for db in self.schema.databases]))
        finally:
            self.client.disconnect()

    def _fast_refresh_schema(self):
        # determine if we can skip expensive table metadata lookups entirely
        # and just worry about finding database names
        # However, with lock-method=auto-detect we must look at table engines
        # to determine what lock method to use
        config = self.config["mysqldump"]
        fast_iterate = (config["lock-method"] != "auto-detect"
                        and not config["exclude-invalid-views"])

        try:
            db_iter = DatabaseIterator(self.client)
            tbl_iter = SimpleTableIterator(self.client, record_engines=True)
            try:
                self.client.connect()
                self.schema.refresh(db_iter=db_iter,
                                    tbl_iter=tbl_iter,
                                    fast_iterate=fast_iterate)
            except MySQLError as exc:
                LOG.debug("MySQLdb error [%d] %s", exc_info=True, *exc.args)
                raise BackupError("MySQL Error [%d] %s" % exc.args)
        finally:
            self.client.disconnect()

    def backup(self):
        """Run a MySQL backup"""
        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        try:
            self.client = connect(self.mysql_config["client"])
        except Exception as ex:
            LOG.debug("%s", ex)
            raise BackupError("Failed connecting to database'")

        if self.dry_run:
            self.mock_env = MockEnvironment()
            self.mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")
            status = self.client.show_databases()
            if not status:
                raise BackupError("Failed to run 'show databases'")
        try:
            if self.config["mysqldump"]["stop-slave"]:
                slave_status = self.client.show_slave_status()
                if not slave_status:
                    raise BackupError("stop-slave enabled, but 'show slave "
                                      "status' failed")
                if slave_status and slave_status["slave_sql_running"] != "Yes":
                    raise BackupError("stop-slave enabled, but replication is "
                                      "not running")
                if not self.dry_run:
                    _stop_slave(self.client, self.config)
            elif self.config["mysqldump"]["bin-log-position"]:
                self.config["mysql:replication"] = {}
                repl_cfg = self.config["mysql:replication"]
                try:
                    master_info = self.client.show_master_status()
                    if master_info:
                        repl_cfg["master_log_file"] = master_info["file"]
                        repl_cfg["master_log_pos"] = master_info["position"]
                except MySQLError as exc:
                    raise BackupError(
                        "Failed to acquire master status [%d] %s" % exc.args)
            self._backup()
        finally:
            if self.config["mysqldump"][
                    "stop-slave"] and "mysql:replication" in self.config:
                _start_slave(self.client, self.config["mysql:replication"])
            if self.mock_env:
                self.mock_env.restore_environment()

    def _backup(self):
        """Real backup method.  May raise BackupError exceptions"""
        config = self.config["mysqldump"]
        # setup defaults_file with ignore-table exclusions
        defaults_file = os.path.join(self.target_directory, "my.cnf")
        write_options(self.mysql_config, defaults_file)
        if config["exclude-invalid-views"]:
            LOG.info("* Finding and excluding invalid views...")
            definitions_path = os.path.join(self.target_directory,
                                            "invalid_views.sql")
            exclude_invalid_views(self.schema, self.client, definitions_path)
        add_exclusions(self.schema, defaults_file)
        # find the path to the mysqldump command
        mysqldump_bin = find_mysqldump(path=config["mysql-binpath"])
        LOG.info("Using mysqldump executable: %s", mysqldump_bin)
        # setup the mysqldump environment
        extra_defaults = config["extra-defaults"]
        try:
            mysqldump = MySQLDump(defaults_file,
                                  mysqldump_bin,
                                  extra_defaults=extra_defaults,
                                  mock_env=self.mock_env)
        except MySQLDumpError as exc:
            raise BackupError(str(exc))
        except Exception as ex:  # pylint: disable=W0703
            LOG.warning(ex)
        mysqldump_version = ".".join(
            [str(digit) for digit in mysqldump.version])
        LOG.info("mysqldump version %s", mysqldump_version)
        options = collect_mysqldump_options(config, mysqldump, self.client)
        validate_mysqldump_options(mysqldump, options)

        os.mkdir(os.path.join(self.target_directory, "backup_data"))

        if (self.config["compression"]["method"] != "none"
                and self.config["compression"]["level"] > 0):
            try:
                _, ext = lookup_compression(
                    self.config["compression"]["method"])
            except OSError as exc:
                raise BackupError(
                    "Unable to load compression method '%s': %s" %
                    (self.config["compression"]["method"], exc))
            LOG.info(
                "Using %s compression level %d with args %s",
                self.config["compression"]["method"],
                self.config["compression"]["level"],
                self.config["compression"]["options"],
            )
        else:
            LOG.info("Not compressing mysqldump output")
            ext = ""

        try:
            start(
                mysqldump=mysqldump,
                schema=self.schema,
                lock_method=config["lock-method"],
                file_per_database=config["file-per-database"],
                open_stream=self._open_stream,
                compression_ext=ext,
                arg_per_database=config["arg-per-database"],
            )
        except MySQLDumpError as exc:
            raise BackupError(str(exc))

    def _open_stream(self, path, mode, method=None):
        """Open a stream through the holland compression api, relative to
        this instance's target directory
        """
        path = str(os.path.join(self.target_directory, "backup_data", path))
        config = deepcopy(self.config["compression"])
        if method:
            config["method"] = method
        stream = open_stream(path, mode, **config)
        return stream

    def info(self):
        """Summarize information about this backup"""

        return (textwrap.dedent("""
        lock-method         = %s
        file-per-database   = %s

        Options used:
        flush-logs          = %s
        flush-privileges    = %s
        routines            = %s
        events              = %s

        Schema Filters:
        databases           = %s
        exclude-databases   = %s
        tables              = %s
        exclude-tables      = %s
        """).strip() % (
            self.config["mysqldump"]["lock-method"],
            self.config["mysqldump"]["file-per-database"] and "yes" or "no",
            self.config["mysqldump"]["flush-logs"],
            self.config["mysqldump"]["flush-privileges"],
            self.config["mysqldump"]["dump-routines"],
            self.config["mysqldump"]["dump-events"],
            ",".join(self.config["mysqldump"]["databases"]),
            ",".join(self.config["mysqldump"]["exclude-databases"]),
            ",".join(self.config["mysqldump"]["tables"]),
            ",".join(self.config["mysqldump"]["exclude-tables"]),
        ))
Beispiel #8
0
class MySQLDumpPlugin(object):
    """MySQLDump Backup Plugin interface for Holland"""

    CONFIGSPEC = CONFIGSPEC

    def __init__(self, name, config, target_directory, dry_run=False):
        self.name = name
        self.config = config
        self.target_directory = target_directory
        self.dry_run = dry_run
        self.config.validate_config(self.CONFIGSPEC)  # -> ValidationError

        # Setup a discovery shell to find schema items
        # This will iterate over items during the estimate
        # or backup phase, which will call schema.refresh()
        self.schema = MySQLSchema()
        config = self.config["mysqldump"]
        self.schema.add_database_filter(include_glob(*config["databases"]))
        self.schema.add_database_filter(exclude_glob(*config["exclude-databases"]))

        self.schema.add_table_filter(include_glob_qualified(*config["tables"]))
        self.schema.add_table_filter(exclude_glob_qualified(*config["exclude-tables"]))
        self.schema.add_engine_filter(include_glob(*config["engines"]))
        self.schema.add_engine_filter(exclude_glob(*config["exclude-engines"]))

        self.mysql_config = build_mysql_config(self.config["mysql:client"])
        self.client = connect(self.mysql_config["client"])

        self.mock_env = None

    def estimate_backup_size(self):
        """Estimate the size of the backup this plugin will generate"""

        LOG.info("Estimating size of mysqldump backup")
        estimate_method = self.config["mysqldump"]["estimate-method"]

        if estimate_method.startswith("const:"):
            try:
                return parse_size(estimate_method[6:])
            except ValueError as exc:
                raise BackupError(str(exc))

        if estimate_method != "plugin":
            raise BackupError("Invalid estimate-method '%s'" % estimate_method)

        try:
            db_iter = DatabaseIterator(self.client)
            tbl_iter = MetadataTableIterator(self.client)
            try:
                self.client.connect()
            except Exception as ex:
                LOG.error("Failed to connect to database")
                LOG.debug("%s", ex)
                raise BackupError("MySQL Error %s" % ex)
            try:
                self.schema.refresh(db_iter=db_iter, tbl_iter=tbl_iter)
            except MySQLError as exc:
                LOG.error("Failed to estimate backup size")
                LOG.debug("[%d] %s", *exc.args)
                raise BackupError("MySQL Error [%d] %s" % exc.args)
            return float(sum([db.size for db in self.schema.databases]))
        finally:
            self.client.disconnect()

    def _fast_refresh_schema(self):
        # determine if we can skip expensive table metadata lookups entirely
        # and just worry about finding database names
        # However, with lock-method=auto-detect we must look at table engines
        # to determine what lock method to use
        config = self.config["mysqldump"]
        fast_iterate = (
            config["lock-method"] != "auto-detect" and not config["exclude-invalid-views"]
        )

        try:
            db_iter = DatabaseIterator(self.client)
            tbl_iter = SimpleTableIterator(self.client, record_engines=True)
            try:
                self.client.connect()
                self.schema.refresh(db_iter=db_iter, tbl_iter=tbl_iter, fast_iterate=fast_iterate)
            except MySQLError as exc:
                LOG.debug("MySQLdb error [%d] %s", exc_info=True, *exc.args)
                raise BackupError("MySQL Error [%d] %s" % exc.args)
        finally:
            self.client.disconnect()

    def backup(self):
        """Run a MySQL backup"""
        if self.schema.timestamp is None:
            self._fast_refresh_schema()

        try:
            self.client = connect(self.mysql_config["client"])
        except Exception as ex:
            LOG.debug("%s", ex)
            raise BackupError("Failed connecting to database'")

        if self.dry_run:
            self.mock_env = MockEnvironment()
            self.mock_env.replace_environment()
            LOG.info("Running in dry-run mode.")
            status = self.client.show_databases()
            if not status:
                raise BackupError("Failed to run 'show databases'")
        try:
            if self.config["mysqldump"]["stop-slave"]:
                slave_status = self.client.show_slave_status()
                if not slave_status:
                    raise BackupError("stop-slave enabled, but 'show slave " "status' failed")
                if slave_status and slave_status["slave_sql_running"] != "Yes":
                    raise BackupError("stop-slave enabled, but replication is " "not running")
                if not self.dry_run:
                    _stop_slave(self.client, self.config)
            elif self.config["mysqldump"]["bin-log-position"]:
                self.config["mysql:replication"] = {}
                repl_cfg = self.config["mysql:replication"]
                try:
                    master_info = self.client.show_master_status()
                    if master_info:
                        repl_cfg["master_log_file"] = master_info["file"]
                        repl_cfg["master_log_pos"] = master_info["position"]
                except MySQLError as exc:
                    raise BackupError("Failed to acquire master status [%d] %s" % exc.args)
            self._backup()
        finally:
            if self.config["mysqldump"]["stop-slave"] and "mysql:replication" in self.config:
                _start_slave(self.client, self.config["mysql:replication"])
            if self.mock_env:
                self.mock_env.restore_environment()

    def _backup(self):
        """Real backup method.  May raise BackupError exceptions"""
        config = self.config["mysqldump"]
        # setup defaults_file with ignore-table exclusions
        defaults_file = os.path.join(self.target_directory, "my.cnf")
        write_options(self.mysql_config, defaults_file)
        if config["exclude-invalid-views"]:
            LOG.info("* Finding and excluding invalid views...")
            definitions_path = os.path.join(self.target_directory, "invalid_views.sql")
            exclude_invalid_views(self.schema, self.client, definitions_path)
        add_exclusions(self.schema, defaults_file)
        # find the path to the mysqldump command
        mysqldump_bin = find_mysqldump(path=config["mysql-binpath"])
        LOG.info("Using mysqldump executable: %s", mysqldump_bin)
        # setup the mysqldump environment
        extra_defaults = config["extra-defaults"]
        try:
            mysqldump = MySQLDump(
                defaults_file, mysqldump_bin, extra_defaults=extra_defaults, mock_env=self.mock_env
            )
        except MySQLDumpError as exc:
            raise BackupError(str(exc))
        except Exception as ex:  # pylint: disable=W0703
            LOG.warning(ex)
        mysqldump_version = ".".join([str(digit) for digit in mysqldump.version])
        LOG.info("mysqldump version %s", mysqldump_version)
        options = collect_mysqldump_options(config, mysqldump, self.client)
        validate_mysqldump_options(mysqldump, options)

        os.mkdir(os.path.join(self.target_directory, "backup_data"))

        if (
            self.config["compression"]["method"] != "none"
            and self.config["compression"]["level"] > 0
        ):
            try:
                _, ext = lookup_compression(self.config["compression"]["method"])
            except OSError as exc:
                raise BackupError(
                    "Unable to load compression method '%s': %s"
                    % (self.config["compression"]["method"], exc)
                )
            LOG.info(
                "Using %s compression level %d with args %s",
                self.config["compression"]["method"],
                self.config["compression"]["level"],
                self.config["compression"]["options"],
            )
        else:
            LOG.info("Not compressing mysqldump output")
            ext = ""

        try:
            start(
                mysqldump=mysqldump,
                schema=self.schema,
                lock_method=config["lock-method"],
                file_per_database=config["file-per-database"],
                open_stream=self._open_stream,
                compression_ext=ext,
                arg_per_database=config["arg-per-database"],
            )
        except MySQLDumpError as exc:
            raise BackupError(str(exc))

    def _open_stream(self, path, mode, method=None):
        """Open a stream through the holland compression api, relative to
        this instance's target directory
        """
        path = str(os.path.join(self.target_directory, "backup_data", path))
        config = deepcopy(self.config["compression"])
        if method:
            config["method"] = method
        stream = open_stream(path, mode, **config)
        return stream

    def info(self):
        """Summarize information about this backup"""
        import textwrap

        return (
            textwrap.dedent(
                """
        lock-method         = %s
        file-per-database   = %s

        Options used:
        flush-logs          = %s
        flush-privileges    = %s
        routines            = %s
        events              = %s

        Schema Filters:
        databases           = %s
        exclude-databases   = %s
        tables              = %s
        exclude-tables      = %s
        """
            ).strip()
            % (
                self.config["mysqldump"]["lock-method"],
                self.config["mysqldump"]["file-per-database"] and "yes" or "no",
                self.config["mysqldump"]["flush-logs"],
                self.config["mysqldump"]["flush-privileges"],
                self.config["mysqldump"]["dump-routines"],
                self.config["mysqldump"]["dump-events"],
                ",".join(self.config["mysqldump"]["databases"]),
                ",".join(self.config["mysqldump"]["exclude-databases"]),
                ",".join(self.config["mysqldump"]["tables"]),
                ",".join(self.config["mysqldump"]["exclude-tables"]),
            )
        )