def _backupDatabase(targetDir,
compressMode,
user,
backupUser,
backupGroup,
database=None):
"""
Backs up an individual PostgreSQL database, or all databases.
This internal method wraps the public method and adds some functionality,
like figuring out a filename, etc.
@param targetDir: Directory into which backups should be written.
@param compressMode: Compress mode to be used for backed-up files.
@param user: User to use for connecting to the database.
@param backupUser: User to own resulting file.
@param backupGroup: Group to own resulting file.
@param database: Name of database, or C{None} for all databases.
@return: Name of the generated backup file.
@raise ValueError: If some value is missing or invalid.
@raise IOError: If there is a problem executing the PostgreSQL dump.
"""
(outputFile, filename) = _getOutputFile(targetDir, database, compressMode)
try:
backupDatabase(user, outputFile, database)
finally:
outputFile.close()
if not os.path.exists(filename):
raise IOError(
"Dump file [%s] does not seem to exist after backup completed." %
filename)
changeOwnership(filename, backupUser, backupGroup)
def _encryptStagingDir(config, local, stagingDir, encryptedDir):
"""
Encrypt a staging directory, creating a new directory in the process.
@param config: Config object.
@param stagingDir: Staging directory to use as source
@param encryptedDir: Target directory into which encrypted files should be written
"""
suCommand = resolveCommand(SU_COMMAND)
files = FilesystemList()
files.addDirContents(stagingDir)
for cleartext in files:
if os.path.isfile(cleartext):
encrypted = "%s%s" % (encryptedDir,
cleartext.replace(stagingDir, ""))
if long(os.stat(cleartext).st_size) == 0:
open(encrypted,
'a').close() # don't bother encrypting empty files
else:
actualCommand = local.amazons3.encryptCommand.replace(
"${input}", cleartext).replace("${output}", encrypted)
subdir = os.path.dirname(encrypted)
if not os.path.isdir(subdir):
os.makedirs(subdir)
changeOwnership(subdir, config.options.backupUser,
config.options.backupGroup)
result = executeCommand(
suCommand,
[config.options.backupUser, "-c", actualCommand])[0]
if result != 0:
raise IOError("Error [%d] encrypting [%s]." %
(result, cleartext))
logger.debug("Completed encrypting staging directory [%s] into [%s]",
stagingDir, encryptedDir)
def _createStagingDirs(config, dailyDir, peers):
"""
Creates staging directories as required.
The main staging directory is the passed in daily directory, something like
C{staging/2002/05/23}. Then, individual peers get their own directories,
i.e. C{staging/2002/05/23/host}.
@param config: Config object.
@param dailyDir: Daily staging directory.
@param peers: List of all configured peers.
@return: Dictionary mapping peer name to staging directory.
"""
mapping = {}
if os.path.isdir(dailyDir):
logger.warn("Staging directory [%s] already existed.", dailyDir)
else:
try:
logger.debug("Creating staging directory [%s].", dailyDir)
os.makedirs(dailyDir)
for path in [ dailyDir, os.path.join(dailyDir, ".."), os.path.join(dailyDir, "..", ".."), ]:
changeOwnership(path, config.options.backupUser, config.options.backupGroup)
except Exception, e:
raise Exception("Unable to create staging directory: %s" % e)
def _dumpFilesystemContents(targetDir, backupUser, backupGroup, compress=True):
"""
Dumps complete listing of filesystem contents via C{ls -laR}.
@param targetDir: Directory to write output file into.
@param backupUser: User which should own the resulting file.
@param backupGroup: Group which should own the resulting file.
@param compress: Indicates whether to compress the output file.
@raise IOError: If the dump fails for some reason.
"""
(outputFile, filename) = _getOutputFile(targetDir, "ls-laR", compress)
try:
# Note: can't count on return status from 'ls', so we don't check it.
command = resolveCommand(LS_COMMAND)
executeCommand(command, [],
returnOutput=False,
ignoreStderr=True,
doNotLog=True,
outputFile=outputFile)
finally:
outputFile.close()
if not os.path.exists(filename):
raise IOError(
"File [%s] does not seem to exist after filesystem contents dump finished."
% filename)
changeOwnership(filename, backupUser, backupGroup)
def _dumpPartitionTable(targetDir, backupUser, backupGroup, compress=True):
"""
Dumps information about the partition table via C{fdisk}.
@param targetDir: Directory to write output file into.
@param backupUser: User which should own the resulting file.
@param backupGroup: Group which should own the resulting file.
@param compress: Indicates whether to compress the output file.
@raise IOError: If the dump fails for some reason.
"""
if not os.path.exists(FDISK_PATH):
logger.info(
"Not executing partition table dump since %s doesn't seem to exist.",
FDISK_PATH)
elif not os.access(FDISK_PATH, os.X_OK):
logger.info(
"Not executing partition table dump since %s cannot be executed.",
FDISK_PATH)
else:
(outputFile, filename) = _getOutputFile(targetDir, "fdisk-l", compress)
try:
command = resolveCommand(FDISK_COMMAND)
result = executeCommand(command, [],
returnOutput=False,
ignoreStderr=True,
outputFile=outputFile)[0]
if result != 0:
raise IOError("Error [%d] executing partition table dump." %
result)
finally:
outputFile.close()
if not os.path.exists(filename):
raise IOError(
"File [%s] does not seem to exist after partition table dump finished."
% filename)
changeOwnership(filename, backupUser, backupGroup)
def _executeBackup(config, backupList, absolutePath, tarfilePath, collectMode, archiveMode, resetDigest, digestPath):
"""
Execute the backup process for the indicated backup list.
This function exists mainly to consolidate functionality between the
L{_collectFile} and L{_collectDirectory} functions. Those functions build
the backup list; this function causes the backup to execute properly and
also manages usage of the digest file on disk as explained in their
comments.
For collect files, the digest file will always just contain the single file
that is being backed up. This might little wasteful in terms of the number
of files that we keep around, but it's consistent and easy to understand.
@param config: Config object.
@param backupList: List to execute backup for
@param absolutePath: Absolute path of directory or file to collect.
@param tarfilePath: Path to tarfile that should be created.
@param collectMode: Collect mode to use.
@param archiveMode: Archive mode to use.
@param resetDigest: Reset digest flag.
@param digestPath: Path to digest file on disk, if needed.
"""
if collectMode != 'incr':
logger.debug("Collect mode is [%s]; no digest will be used.", collectMode)
if len(backupList) == 1 and backupList[0] == absolutePath: # special case for individual file
logger.info("Backing up file [%s] (%s).", absolutePath, displayBytes(backupList.totalSize()))
else:
logger.info("Backing up %d files in [%s] (%s).", len(backupList), absolutePath, displayBytes(backupList.totalSize()))
if len(backupList) > 0:
backupList.generateTarfile(tarfilePath, archiveMode, True)
changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
else:
if resetDigest:
logger.debug("Based on resetDigest flag, digest will be cleared.")
oldDigest = {}
else:
logger.debug("Based on resetDigest flag, digest will loaded from disk.")
oldDigest = _loadDigest(digestPath)
(removed, newDigest) = backupList.removeUnchanged(oldDigest, captureDigest=True)
logger.debug("Removed %d unchanged files based on digest values.", removed)
if len(backupList) == 1 and backupList[0] == absolutePath: # special case for individual file
logger.info("Backing up file [%s] (%s).", absolutePath, displayBytes(backupList.totalSize()))
else:
logger.info("Backing up %d files in [%s] (%s).", len(backupList), absolutePath, displayBytes(backupList.totalSize()))
if len(backupList) > 0:
backupList.generateTarfile(tarfilePath, archiveMode, True)
changeOwnership(tarfilePath, config.options.backupUser, config.options.backupGroup)
_writeDigest(config, newDigest, digestPath)
def writeIndicatorFile(targetDir, indicatorFile, backupUser, backupGroup):
"""
Writes an indicator file into a target directory.
@param targetDir: Target directory in which to write indicator
@param indicatorFile: Name of the indicator file
@param backupUser: User that indicator file should be owned by
@param backupGroup: Group that indicator file should be owned by
@raise IOException: If there is a problem writing the indicator file
"""
filename = os.path.join(targetDir, indicatorFile)
logger.debug("Writing indicator file [%s].", filename)
try:
open(filename, "w").write("")
changeOwnership(filename, backupUser, backupGroup)
except Exception, e:
logger.error("Error writing [%s]: %s", filename, e)
raise e
def _writeDigest(config, digest, digestPath):
"""
Writes the digest dictionary to the indicated digest path on disk.
If we can't write the digest successfully for any reason, we'll log the
condition but won't throw an exception.
@param config: Config object.
@param digest: Digest dictionary to write to disk.
@param digestPath: Path to the digest file on disk.
"""
try:
pickle.dump(digest, open(digestPath, "w"))
changeOwnership(digestPath, config.options.backupUser, config.options.backupGroup)
logger.debug("Wrote new digest [%s] to disk: %d entries.", digestPath, len(digest))
except:
logger.error("Failed to write digest [%s] to disk.", digestPath)
def _encryptFile(sourcePath,
encryptMode,
encryptTarget,
backupUser,
backupGroup,
removeSource=False):
"""
Encrypts the source file using the indicated mode.
The encrypted file will be owned by the indicated backup user and group. If
C{removeSource} is C{True}, then the source file will be removed after it is
successfully encrypted.
Currently, only the C{"gpg"} encrypt mode is supported.
@param sourcePath: Absolute path of the source file to encrypt
@param encryptMode: Encryption mode (only "gpg" is allowed)
@param encryptTarget: Encryption target (GPG recipient)
@param backupUser: User that target files should be owned by
@param backupGroup: Group that target files should be owned by
@param removeSource: Indicates whether to remove the source file
@return: Path to the newly-created encrypted file.
@raise ValueError: If an invalid encrypt mode is passed in.
@raise IOError: If there is a problem accessing, encrypting or removing the source file.
"""
if not os.path.exists(sourcePath):
raise ValueError("Source path [%s] does not exist." % sourcePath)
if encryptMode == 'gpg':
encryptedPath = _encryptFileWithGpg(sourcePath,
recipient=encryptTarget)
else:
raise ValueError("Unknown encrypt mode [%s]" % encryptMode)
changeOwnership(encryptedPath, backupUser, backupGroup)
if removeSource:
if os.path.exists(sourcePath):
try:
os.remove(sourcePath)
logger.debug("Completed removing old file [%s].", sourcePath)
except:
raise IOError(
"Failed to remove file [%s] after encrypting it." %
(sourcePath))
return encryptedPath
def _writeToAmazonS3(config, local, stagingDirs):
"""
Writes the indicated staging directories to an Amazon S3 bucket.
Each of the staging directories listed in C{stagingDirs} will be written to
the configured Amazon S3 bucket from local configuration. The directories
will be placed into the image at the root by date, so staging directory
C{/opt/stage/2005/02/10} will be placed into the S3 bucket at C{/2005/02/10}.
If an encrypt commmand is provided, the files will be encrypted first.
@param config: Config object.
@param local: Local config object.
@param stagingDirs: Dictionary mapping directory path to date suffix.
@raise ValueError: Under many generic error conditions
@raise IOError: If there is a problem writing to Amazon S3
"""
for stagingDir in stagingDirs.keys():
logger.debug("Storing stage directory to Amazon S3 [%s].", stagingDir)
dateSuffix = stagingDirs[stagingDir]
s3BucketUrl = "s3://%s/%s" % (local.amazons3.s3Bucket, dateSuffix)
logger.debug("S3 bucket URL is [%s]", s3BucketUrl)
_clearExistingBackup(config, s3BucketUrl)
if local.amazons3.encryptCommand is None:
logger.debug(
"Encryption is disabled; files will be uploaded in cleartext.")
_uploadStagingDir(config, stagingDir, s3BucketUrl)
_verifyUpload(config, stagingDir, s3BucketUrl)
else:
logger.debug(
"Encryption is enabled; files will be uploaded after being encrypted."
)
encryptedDir = tempfile.mkdtemp(dir=config.options.workingDir)
changeOwnership(encryptedDir, config.options.backupUser,
config.options.backupGroup)
try:
_encryptStagingDir(config, local, stagingDir, encryptedDir)
_uploadStagingDir(config, encryptedDir, s3BucketUrl)
_verifyUpload(config, encryptedDir, s3BucketUrl)
finally:
if os.path.exists(encryptedDir):
shutil.rmtree(encryptedDir)
def _dumpDebianPackages(targetDir, backupUser, backupGroup, compress=True):
"""
Dumps a list of currently installed Debian packages via C{dpkg}.
@param targetDir: Directory to write output file into.
@param backupUser: User which should own the resulting file.
@param backupGroup: Group which should own the resulting file.
@param compress: Indicates whether to compress the output file.
@raise IOError: If the dump fails for some reason.
"""
if not os.path.exists(DPKG_PATH):
logger.info(
"Not executing Debian package dump since %s doesn't seem to exist.",
DPKG_PATH)
elif not os.access(DPKG_PATH, os.X_OK):
logger.info(
"Not executing Debian package dump since %s cannot be executed.",
DPKG_PATH)
else:
(outputFile, filename) = _getOutputFile(targetDir, "dpkg-selections",
compress)
try:
command = resolveCommand(DPKG_COMMAND)
result = executeCommand(command, [],
returnOutput=False,
ignoreStderr=True,
doNotLog=True,
outputFile=outputFile)[0]
if result != 0:
raise IOError("Error [%d] executing Debian package dump." %
result)
finally:
outputFile.close()
if not os.path.exists(filename):
raise IOError(
"File [%s] does not seem to exist after Debian package dump finished."
% filename)
changeOwnership(filename, backupUser, backupGroup)
logger.debug("Creating staging directory [%s].", dailyDir)
os.makedirs(dailyDir)
for path in [ dailyDir, os.path.join(dailyDir, ".."), os.path.join(dailyDir, "..", ".."), ]:
changeOwnership(path, config.options.backupUser, config.options.backupGroup)
except Exception, e:
raise Exception("Unable to create staging directory: %s" % e)
for peer in peers:
peerDir = os.path.join(dailyDir, peer.name)
mapping[peer.name] = peerDir
if os.path.isdir(peerDir):
logger.warn("Peer staging directory [%s] already existed.", peerDir)
else:
try:
logger.debug("Creating peer staging directory [%s].", peerDir)
os.makedirs(peerDir)
changeOwnership(peerDir, config.options.backupUser, config.options.backupGroup)
except Exception, e:
raise Exception("Unable to create staging directory: %s" % e)
return mapping
########################################################################
# Private attribute "getter" functions
########################################################################
####################################
# _getIgnoreFailuresFlag() function
####################################
def _getIgnoreFailuresFlag(options, config, peer):
"""
def _splitFile(sourcePath,
splitSize,
backupUser,
backupGroup,
removeSource=False):
"""
Splits the source file into chunks of the indicated size.
The split files will be owned by the indicated backup user and group. If
C{removeSource} is C{True}, then the source file will be removed after it is
successfully split.
@param sourcePath: Absolute path of the source file to split
@param splitSize: Encryption mode (only "gpg" is allowed)
@param backupUser: User that target files should be owned by
@param backupGroup: Group that target files should be owned by
@param removeSource: Indicates whether to remove the source file
@raise IOError: If there is a problem accessing, splitting or removing the source file.
"""
cwd = os.getcwd()
try:
if not os.path.exists(sourcePath):
raise ValueError("Source path [%s] does not exist." % sourcePath)
dirname = os.path.dirname(sourcePath)
filename = os.path.basename(sourcePath)
prefix = "%s_" % filename
bytes = int(splitSize.bytes) # pylint: disable=W0622
os.chdir(
dirname
) # need to operate from directory that we want files written to
command = resolveCommand(SPLIT_COMMAND)
args = [
"--verbose",
"--numeric-suffixes",
"--suffix-length=5",
"--bytes=%d" % bytes,
filename,
prefix,
]
(result, output) = executeCommand(command,
args,
returnOutput=True,
ignoreStderr=False)
if result != 0:
raise IOError("Error [%d] calling split for [%s]." %
(result, sourcePath))
pattern = re.compile(r"(creating file [`'])(%s)(.*)(')" % prefix)
match = pattern.search(output[-1:][0])
if match is None:
raise IOError("Unable to parse output from split command.")
value = int(match.group(3).strip())
for index in range(0, value):
path = "%s%05d" % (prefix, index)
if not os.path.exists(path):
raise IOError(
"After call to split, expected file [%s] does not exist." %
path)
changeOwnership(path, backupUser, backupGroup)
if removeSource:
if os.path.exists(sourcePath):
try:
os.remove(sourcePath)
logger.debug("Completed removing old file [%s].",
sourcePath)
except:
raise IOError(
"Failed to remove file [%s] after splitting it." %
(sourcePath))
finally:
os.chdir(cwd)