class Archive:
    def __init__(self,
                 backup_dir,
                 output_file,
                 no_gzip=False,
                 verbose=False,
                 binary="tar"):
        self.backup_dir = backup_dir
        self.output_file = output_file
        self.no_gzip = no_gzip
        self.verbose = verbose
        self.binary = binary

        self._command = None

        signal(SIGINT, self.close)
        signal(SIGTERM, self.close)

    def close(self, exit_code=None, frame=None):
        if self._command:
            logging.debug("Killing running subprocess/command: %s" %
                          self._command.command)
            self._command.close()

    def run(self):
        if os.path.isdir(self.backup_dir):
            if not os.path.isfile(self.output_file):
                try:
                    backup_base_dir = os.path.dirname(self.backup_dir)
                    backup_base_name = os.path.basename(self.backup_dir)

                    log_msg = "Archiving and compressing directory: %s" % self.backup_dir
                    cmd_flags = [
                        "-C", backup_base_dir, "-czf", self.output_file,
                        "--remove-files", backup_base_name
                    ]

                    if self.no_gzip:
                        log_msg = "Archiving directory: %s" % self.backup_dir
                        cmd_flags = [
                            "-C", backup_base_dir, "-cf", self.output_file,
                            "--remove-files", backup_base_name
                        ]

                    try:
                        logging.info(log_msg)
                        self._command = LocalCommand(self.binary, cmd_flags,
                                                     self.verbose)
                        self._command.run()
                    except Exception, e:
                        raise e
                except Exception, e:
                    logging.fatal("Failed archiving file: %s! Error: %s" %
                                  (self.output_file, e))
                    raise e
            elif os.path.isfile(self.output_file):
                logging.fatal("Output file: %s already exists!" %
                              self.output_file)
                raise Exception, "Output file %s already exists!" % self.output_file, None
class Archive:
    def __init__(self, backup_dir, output_file, no_gzip=False, verbose=False, binary="tar"):
        self.backup_dir  = backup_dir
        self.output_file = output_file
        self.no_gzip     = no_gzip
        self.verbose     = verbose
        self.binary      = binary

        self._command  = None

        signal(SIGINT, self.close)
        signal(SIGTERM, self.close)

    def close(self, exit_code=None, frame=None):
        if self._command:
            logging.debug("Killing running subprocess/command: %s" % self._command.command)
            del exit_code
            del frame
            self._command.close()

    def run(self):
        if os.path.isdir(self.backup_dir):
            if not os.path.isfile(self.output_file):
                try:
                    backup_base_dir  = os.path.dirname(self.backup_dir)
                    backup_base_name = os.path.basename(self.backup_dir)

                    log_msg   = "Archiving and compressing directory: %s" % self.backup_dir
                    cmd_flags = ["-C", backup_base_dir, "-czf", self.output_file, "--remove-files", backup_base_name]

                    if self.no_gzip:
                        log_msg   = "Archiving directory: %s" % self.backup_dir
                        cmd_flags = ["-C", backup_base_dir, "-cf", self.output_file, "--remove-files", backup_base_name]

                    try:
                        logging.info(log_msg)
                        self._command = LocalCommand(self.binary, cmd_flags, self.verbose)
                        self._command.run()
                    except Exception, e:
                        raise e
                except Exception, e:
                    logging.fatal("Failed archiving file: %s! Error: %s" % (self.output_file, e))
                    raise e
            elif os.path.isfile(self.output_file):
                logging.fatal("Output file: %s already exists!" % self.output_file)
                raise Exception, "Output file %s already exists!" % self.output_file, None
class Mongodump(Process):
    def __init__(self, response_queue, backup_name, host_port, user, password, authdb, base_dir, binary,
                 dump_gzip=False, verbose=False):
        Process.__init__(self)
        self.host, port     = host_port.split(":")
        self.host_port      = host_port
        self.port           = int(port)
        self.response_queue = response_queue
        self.backup_name    = backup_name
        self.user           = user
        self.password       = password
        self.authdb         = authdb
        self.base_dir       = base_dir
        self.binary         = binary
        self.dump_gzip      = dump_gzip
        self.verbose        = verbose

        self._command   = None
        self.completed  = False 
        self.backup_dir = "%s/%s" % (self.base_dir, self.backup_name)
        self.dump_dir   = "%s/dump" % self.backup_dir
        self.oplog_file = "%s/oplog.bson" % self.dump_dir
        self.start_time = time()

        signal(SIGINT, self.close)
        signal(SIGTERM, self.close)

    def close(self, exit_code=None, frame=None):
        if self._command:
            logging.debug("Killing running subprocess/command: %s" % self._command.command)
            self._command.close()

    def run(self):
        logging.info("Starting mongodump (with oplog) backup of %s/%s:%i" % (
            self.backup_name,
            self.host,
            self.port
        ))

        mongodump_flags = ["-h", self.host_port, "--oplog", "-o", "%s/dump" % self.backup_dir]
        if self.dump_gzip:
            logging.debug("Enabling inline mongodump compression using --gzip flag")
            mongodump_flags.extend(["--gzip"])
        if self.authdb and self.authdb != "admin":
            logging.debug("Using database %s for authentication" % self.authdb)
            mongodump_flags.extend(["--authenticationDatabase", self.authdb])
        if self.user and self.password:
            mongodump_flags.extend(["-u", self.user, "-p", self.password])

        try:
            commands = []
            if os.path.isdir(self.dump_dir):
                commands.append(["rm", ["-rf", self.dump_dir]])
            commands.append(["mkdir", ["-p", self.dump_dir]])
            commands.append([self.binary, mongodump_flags])

            for (command, command_flags) in commands:
                self._command = LocalCommand(command, command_flags, self.verbose)
                self._command.run()
        except Exception, e:
            logging.error("Error performing mongodump: %s" % e)
            return None

        oplog = OplogInfo(self.oplog_file, self.dump_gzip)
        self.completed = True
        self.response_queue.put({
            'host': self.host,
            'port': self.port,
            'file': self.oplog_file,
            'count': oplog.count(),
            'last_ts': oplog.last_ts(),
            'first_ts': oplog.first_ts(),
            'completed': self.completed
        })

        time_diff = time() - self.start_time
        logging.info("Backup for %s/%s:%s completed in %s sec with %i oplog changes captured to: %s" % (
            self.backup_name, self.host, self.port, time_diff, oplog.count(), str(oplog.last_ts())
        ))
Esempio n. 4
0
class Mongodump(Process):
    def __init__(self,
                 response_queue,
                 backup_name,
                 host_port,
                 user,
                 password,
                 authdb,
                 base_dir,
                 binary,
                 dump_gzip=False,
                 verbose=False):
        Process.__init__(self)
        self.host, port = host_port.split(":")
        self.host_port = host_port
        self.port = int(port)
        self.response_queue = response_queue
        self.backup_name = backup_name
        self.user = user
        self.password = password
        self.authdb = authdb
        self.base_dir = base_dir
        self.binary = binary
        self.dump_gzip = dump_gzip
        self.verbose = verbose

        self._command = None
        self.completed = False
        self.backup_dir = "%s/%s" % (self.base_dir, self.backup_name)
        self.dump_dir = "%s/dump" % self.backup_dir
        self.oplog_file = "%s/oplog.bson" % self.dump_dir
        self.start_time = time()

        signal(SIGINT, self.close)
        signal(SIGTERM, self.close)

    def close(self, exit_code=None, frame=None):
        if self._command:
            logging.debug("Killing running subprocess/command: %s" %
                          self._command.command)
            self._command.close()

    def run(self):
        logging.info("Starting mongodump (with oplog) backup of %s/%s:%i" %
                     (self.backup_name, self.host, self.port))

        mongodump_flags = [
            "-h", self.host_port, "--oplog", "-o",
            "%s/dump" % self.backup_dir
        ]
        if self.dump_gzip:
            logging.debug(
                "Enabling inline mongodump compression using --gzip flag")
            mongodump_flags.extend(["--gzip"])
        if self.authdb and self.authdb != "admin":
            logging.debug("Using database %s for authentication" % self.authdb)
            mongodump_flags.extend(["--authenticationDatabase", self.authdb])
        if self.user and self.password:
            mongodump_flags.extend(["-u", self.user, "-p", self.password])

        try:
            commands = []
            if os.path.isdir(self.dump_dir):
                commands.append(["rm", ["-rf", self.dump_dir]])
            commands.append(["mkdir", ["-p", self.dump_dir]])
            commands.append([self.binary, mongodump_flags])

            for (command, command_flags) in commands:
                self._command = LocalCommand(command, command_flags,
                                             self.verbose)
                self._command.run()
        except Exception, e:
            logging.error("Error performing mongodump: %s" % e)
            return None

        oplog = OplogInfo(self.oplog_file, self.dump_gzip)
        self.completed = True
        self.response_queue.put({
            'host': self.host,
            'port': self.port,
            'file': self.oplog_file,
            'count': oplog.count(),
            'last_ts': oplog.last_ts(),
            'first_ts': oplog.first_ts(),
            'completed': self.completed
        })

        time_diff = time() - self.start_time
        logging.info(
            "Backup for %s/%s:%s completed in %s sec with %i oplog changes captured to: %s"
            % (self.backup_name, self.host, self.port, time_diff,
               oplog.count(), str(oplog.last_ts())))