class Archive:
def __init__(self,
backup_dir,
output_file,
no_gzip=False,
verbose=False,
binary="tar"):
self.backup_dir = backup_dir
self.output_file = output_file
self.no_gzip = no_gzip
self.verbose = verbose
self.binary = binary
self._command = None
signal(SIGINT, self.close)
signal(SIGTERM, self.close)
def close(self, exit_code=None, frame=None):
if self._command:
logging.debug("Killing running subprocess/command: %s" %
self._command.command)
self._command.close()
def run(self):
if os.path.isdir(self.backup_dir):
if not os.path.isfile(self.output_file):
try:
backup_base_dir = os.path.dirname(self.backup_dir)
backup_base_name = os.path.basename(self.backup_dir)
log_msg = "Archiving and compressing directory: %s" % self.backup_dir
cmd_flags = [
"-C", backup_base_dir, "-czf", self.output_file,
"--remove-files", backup_base_name
]
if self.no_gzip:
log_msg = "Archiving directory: %s" % self.backup_dir
cmd_flags = [
"-C", backup_base_dir, "-cf", self.output_file,
"--remove-files", backup_base_name
]
try:
logging.info(log_msg)
self._command = LocalCommand(self.binary, cmd_flags,
self.verbose)
self._command.run()
except Exception, e:
raise e
except Exception, e:
logging.fatal("Failed archiving file: %s! Error: %s" %
(self.output_file, e))
raise e
elif os.path.isfile(self.output_file):
logging.fatal("Output file: %s already exists!" %
self.output_file)
raise Exception, "Output file %s already exists!" % self.output_file, None
class Archive:
def __init__(self, backup_dir, output_file, no_gzip=False, verbose=False, binary="tar"):
self.backup_dir = backup_dir
self.output_file = output_file
self.no_gzip = no_gzip
self.verbose = verbose
self.binary = binary
self._command = None
signal(SIGINT, self.close)
signal(SIGTERM, self.close)
def close(self, exit_code=None, frame=None):
if self._command:
logging.debug("Killing running subprocess/command: %s" % self._command.command)
del exit_code
del frame
self._command.close()
def run(self):
if os.path.isdir(self.backup_dir):
if not os.path.isfile(self.output_file):
try:
backup_base_dir = os.path.dirname(self.backup_dir)
backup_base_name = os.path.basename(self.backup_dir)
log_msg = "Archiving and compressing directory: %s" % self.backup_dir
cmd_flags = ["-C", backup_base_dir, "-czf", self.output_file, "--remove-files", backup_base_name]
if self.no_gzip:
log_msg = "Archiving directory: %s" % self.backup_dir
cmd_flags = ["-C", backup_base_dir, "-cf", self.output_file, "--remove-files", backup_base_name]
try:
logging.info(log_msg)
self._command = LocalCommand(self.binary, cmd_flags, self.verbose)
self._command.run()
except Exception, e:
raise e
except Exception, e:
logging.fatal("Failed archiving file: %s! Error: %s" % (self.output_file, e))
raise e
elif os.path.isfile(self.output_file):
logging.fatal("Output file: %s already exists!" % self.output_file)
raise Exception, "Output file %s already exists!" % self.output_file, None
class Mongodump(Process):
def __init__(self, response_queue, backup_name, host_port, user, password, authdb, base_dir, binary,
dump_gzip=False, verbose=False):
Process.__init__(self)
self.host, port = host_port.split(":")
self.host_port = host_port
self.port = int(port)
self.response_queue = response_queue
self.backup_name = backup_name
self.user = user
self.password = password
self.authdb = authdb
self.base_dir = base_dir
self.binary = binary
self.dump_gzip = dump_gzip
self.verbose = verbose
self._command = None
self.completed = False
self.backup_dir = "%s/%s" % (self.base_dir, self.backup_name)
self.dump_dir = "%s/dump" % self.backup_dir
self.oplog_file = "%s/oplog.bson" % self.dump_dir
self.start_time = time()
signal(SIGINT, self.close)
signal(SIGTERM, self.close)
def close(self, exit_code=None, frame=None):
if self._command:
logging.debug("Killing running subprocess/command: %s" % self._command.command)
self._command.close()
def run(self):
logging.info("Starting mongodump (with oplog) backup of %s/%s:%i" % (
self.backup_name,
self.host,
self.port
))
mongodump_flags = ["-h", self.host_port, "--oplog", "-o", "%s/dump" % self.backup_dir]
if self.dump_gzip:
logging.debug("Enabling inline mongodump compression using --gzip flag")
mongodump_flags.extend(["--gzip"])
if self.authdb and self.authdb != "admin":
logging.debug("Using database %s for authentication" % self.authdb)
mongodump_flags.extend(["--authenticationDatabase", self.authdb])
if self.user and self.password:
mongodump_flags.extend(["-u", self.user, "-p", self.password])
try:
commands = []
if os.path.isdir(self.dump_dir):
commands.append(["rm", ["-rf", self.dump_dir]])
commands.append(["mkdir", ["-p", self.dump_dir]])
commands.append([self.binary, mongodump_flags])
for (command, command_flags) in commands:
self._command = LocalCommand(command, command_flags, self.verbose)
self._command.run()
except Exception, e:
logging.error("Error performing mongodump: %s" % e)
return None
oplog = OplogInfo(self.oplog_file, self.dump_gzip)
self.completed = True
self.response_queue.put({
'host': self.host,
'port': self.port,
'file': self.oplog_file,
'count': oplog.count(),
'last_ts': oplog.last_ts(),
'first_ts': oplog.first_ts(),
'completed': self.completed
})
time_diff = time() - self.start_time
logging.info("Backup for %s/%s:%s completed in %s sec with %i oplog changes captured to: %s" % (
self.backup_name, self.host, self.port, time_diff, oplog.count(), str(oplog.last_ts())
))
class Mongodump(Process):
def __init__(self,
response_queue,
backup_name,
host_port,
user,
password,
authdb,
base_dir,
binary,
dump_gzip=False,
verbose=False):
Process.__init__(self)
self.host, port = host_port.split(":")
self.host_port = host_port
self.port = int(port)
self.response_queue = response_queue
self.backup_name = backup_name
self.user = user
self.password = password
self.authdb = authdb
self.base_dir = base_dir
self.binary = binary
self.dump_gzip = dump_gzip
self.verbose = verbose
self._command = None
self.completed = False
self.backup_dir = "%s/%s" % (self.base_dir, self.backup_name)
self.dump_dir = "%s/dump" % self.backup_dir
self.oplog_file = "%s/oplog.bson" % self.dump_dir
self.start_time = time()
signal(SIGINT, self.close)
signal(SIGTERM, self.close)
def close(self, exit_code=None, frame=None):
if self._command:
logging.debug("Killing running subprocess/command: %s" %
self._command.command)
self._command.close()
def run(self):
logging.info("Starting mongodump (with oplog) backup of %s/%s:%i" %
(self.backup_name, self.host, self.port))
mongodump_flags = [
"-h", self.host_port, "--oplog", "-o",
"%s/dump" % self.backup_dir
]
if self.dump_gzip:
logging.debug(
"Enabling inline mongodump compression using --gzip flag")
mongodump_flags.extend(["--gzip"])
if self.authdb and self.authdb != "admin":
logging.debug("Using database %s for authentication" % self.authdb)
mongodump_flags.extend(["--authenticationDatabase", self.authdb])
if self.user and self.password:
mongodump_flags.extend(["-u", self.user, "-p", self.password])
try:
commands = []
if os.path.isdir(self.dump_dir):
commands.append(["rm", ["-rf", self.dump_dir]])
commands.append(["mkdir", ["-p", self.dump_dir]])
commands.append([self.binary, mongodump_flags])
for (command, command_flags) in commands:
self._command = LocalCommand(command, command_flags,
self.verbose)
self._command.run()
except Exception, e:
logging.error("Error performing mongodump: %s" % e)
return None
oplog = OplogInfo(self.oplog_file, self.dump_gzip)
self.completed = True
self.response_queue.put({
'host': self.host,
'port': self.port,
'file': self.oplog_file,
'count': oplog.count(),
'last_ts': oplog.last_ts(),
'first_ts': oplog.first_ts(),
'completed': self.completed
})
time_diff = time() - self.start_time
logging.info(
"Backup for %s/%s:%s completed in %s sec with %i oplog changes captured to: %s"
% (self.backup_name, self.host, self.port, time_diff,
oplog.count(), str(oplog.last_ts())))