def __init__(self, storageDirFd, testModeFlag, sourceUrl):
"""Create a temporary storage file and a handle to it."""
self.testModeFlag = testModeFlag
self.sourceUrl = sourceUrl
guerillabackup.assertSourceUrlSpecificationConforming(sourceUrl)
self.elementIdParts = DefaultFileSystemSink.internalGetElementIdParts(
sourceUrl, None)
self.storageDirFd = None
if self.elementIdParts[0] == '':
self.storageDirFd = os.dup(storageDirFd)
else:
self.storageDirFd = guerillabackup.secureOpenAt(
storageDirFd,
self.elementIdParts[0][1:],
symlinksAllowedFlag=False,
dirOpenFlags=os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
| os.O_NOCTTY,
dirCreateMode=0o700,
fileOpenFlags=os.O_DIRECTORY | os.O_RDONLY | os.O_NOFOLLOW
| os.O_CREAT | os.O_EXCL | os.O_NOCTTY,
fileCreateMode=0o700)
# Generate a temporary file name in the same directory.
while True:
self.tmpFileName = 'tmp-%s-%d' % (self.elementIdParts[1],
random.randint(0, 1 << 30))
try:
self.streamFd = guerillabackup.secureOpenAt(
self.storageDirFd,
self.tmpFileName,
symlinksAllowedFlag=False,
dirOpenFlags=os.O_RDONLY | os.O_DIRECTORY | os.O_NOFOLLOW
| os.O_NOCTTY,
dirCreateMode=None,
fileOpenFlags=os.O_RDWR | os.O_NOFOLLOW | os.O_CREAT
| os.O_EXCL | os.O_NOCTTY,
fileCreateMode=0o600)
break
except OSError as openError:
if openError.errno != errno.EEXIST:
os.close(self.storageDirFd)
raise
def __init__(self, storageDirFd, elementId):
"""Create a file based backup data element and make sure the
storage files are at least accessible without reading or validating
the content."""
# Extract the source URL from the elementId.
fileNameSepPos = elementId.rfind('/')
if (fileNameSepPos < 0) or (elementId[0] != '/'):
raise Exception('Invalid elementId without a separator')
lastNameStart = elementId.find('-', fileNameSepPos)
lastNameEnd = elementId.rfind('-')
if ((lastNameStart < 0) or (lastNameEnd < 0) or
(lastNameStart+1 >= lastNameEnd)):
raise Exception('Malformed last name in elementId')
self.sourceUrl = elementId[:fileNameSepPos+1]+elementId[lastNameStart+1:lastNameEnd]
guerillabackup.assertSourceUrlSpecificationConforming(self.sourceUrl)
# Now try to create the StorageBackupDataElementInterface element.
self.storageDirFd = storageDirFd
# Just stat the data and info file, that are mandatory.
os.stat('.'+elementId+'.data', dir_fd=self.storageDirFd)
os.stat('.'+elementId+'.info', dir_fd=self.storageDirFd)
self.elementId = elementId
# Cache the metainfo once loaded.
self.metaInfo = None
def __init__(self, descriptionTuple):
"""Initialize a single input description using a 5-value tuple,
e.g. extracted directly from the CONFIG_INPUT_LIST_KEY parameter.
@param descriptionTuple the tuple, the meaning of the 5 values
to be extracted is:
* Input directory: directory to search for logfiles
* Input file regex: regular expression to select compressed
or uncompressed logfiles for inclusion.
* Source URL transformation: If None, the first named group
of the "input file regex" is used as source URL. When not
starting with a "/", the transformation string is the name
to include literally in the URL after the "input directory"
name.
* Policy: If not none, include this string as handling policy
within the manifest.
* Encryption key name: If not None, encrypt the input using
the named key."""
# Accept list also.
if ((not isinstance(descriptionTuple, tuple))
and (not isinstance(descriptionTuple, list))):
raise Exception('Input description has to be list or tuple')
if len(descriptionTuple) != 5:
raise Exception(
'Input description has to be tuple with 5 elements')
self.inputDirectoryName = os.path.normpath(descriptionTuple[0])
# "//..." is a normalized path, get rid of double slashes.
self.sourceUrlPath = self.inputDirectoryName.replace('//', '/')
if self.sourceUrlPath[-1] != '/':
self.sourceUrlPath += '/'
self.inputFileRegex = re.compile(descriptionTuple[1])
self.sourceTransformationPattern = descriptionTuple[2]
try:
if self.sourceTransformationPattern is None:
guerillabackup.assertSourceUrlSpecificationConforming(
self.sourceUrlPath + 'testname')
elif self.sourceTransformationPattern[0] != '/':
guerillabackup.assertSourceUrlSpecificationConforming(
self.sourceUrlPath + self.sourceTransformationPattern)
else:
guerillabackup.assertSourceUrlSpecificationConforming(
self.sourceTransformationPattern)
except Exception as assertException:
raise Exception('Source URL transformation malformed: ' +
assertException.args[0])
self.handlingPolicyName = descriptionTuple[3]
self.encryptionKeyName = descriptionTuple[4]
def getBackupDataElementForMetaData(self, sourceUrl, metaData):
"""Retrieve a single stored backup data element from the storage.
@param sourceUrl the URL identifying the source that produced
the stored data elements.
@param metaData metaData dictionary for the element of interest.
@throws Exception when an incompatible query, update or read
is in progress.
@return the element or None if no matching element was found."""
# At first get an iterator over all elements in file system that
# might match the given query.
guerillabackup.assertSourceUrlSpecificationConforming(sourceUrl)
elementIdParts = \
guerillabackup.DefaultFileSystemSink.internalGetElementIdParts(
sourceUrl, metaData)
# Now search the directory for all files conforming to the specifiction.
# As there may exist multiple files with the same time stamp and
# type, load also the meta data and check if matches the query.
elementDirFd = None
if len(elementIdParts[0]) == 0:
elementDirFd = os.dup(self.storageDirFd)
else:
try:
elementDirFd = guerillabackup.secureOpenAt(
self.storageDirFd, elementIdParts[0][1:], symlinksAllowedFlag=False,
dirOpenFlags=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW|os.O_NOCTTY,
dirCreateMode=0o700,
fileOpenFlags=os.O_DIRECTORY|os.O_RDONLY|os.O_NOFOLLOW|os.O_CREAT|os.O_EXCL|os.O_NOCTTY)
except OSError as dirOpenError:
# Directory does not exist, so there cannot be any valid element.
if dirOpenError.errno == errno.ENOENT:
return None
raise
searchPrefix = elementIdParts[2]
searchSuffix = '-%s-%s.data' % (elementIdParts[1], elementIdParts[3])
result = None
try:
fileList = guerillabackup.listDirAt(elementDirFd)
for fileName in fileList:
if ((not fileName.startswith(searchPrefix)) or
(not fileName.endswith(searchSuffix))):
continue
# Just verify, that the serial part is really an integer but no
# need to handle the exception. This would indicate storage corruption,
# so we need to stop anyway.
serialStr = fileName[len(searchPrefix):-len(searchSuffix)]
if serialStr != '':
int(serialStr)
# So file might match, load the meta data.
metaDataFd = -1
fileMetaInfo = None
try:
metaDataFd = guerillabackup.secureOpenAt(
elementDirFd, './%s.info' % fileName[:-5],
symlinksAllowedFlag=False,
dirOpenFlags=os.O_RDONLY|os.O_DIRECTORY|os.O_NOFOLLOW|os.O_NOCTTY,
dirCreateMode=None,
fileOpenFlags=os.O_RDONLY|os.O_NOFOLLOW|os.O_NOCTTY)
metaInfoData = guerillabackup.readFully(metaDataFd)
fileMetaInfo = BackupElementMetainfo.unserialize(metaInfoData)
finally:
if metaDataFd >= 0:
os.close(metaDataFd)
if fileMetaInfo.get('DataUuid') != metaData.get('DataUuid'):
continue
elementId = '%s/%s' % (elementIdParts[0], fileName[:-5])
result = FileStorageBackupDataElement(self.storageDirFd, elementId)
break
finally:
os.close(elementDirFd)
return result
def __init__(self, sourceUrl, descriptionDict):
"""Initialize a single tar backup description using values
from a dictionary, e.g. extracted directly from the CONFIG_LIST_KEY
parameter.
@param sourceUrl the URL which will identify the backups from
this subunit. It is used also to store persistency information
for that unit.
@param descriptionDict dictionary containing the parameters
for a single tar backup task.
* PreBackupCommand: execute this command given as list of
arguments before starting the backup, e.g. create a filesystem
or virtual machine snapshot, perform cleanup.
* PostBackupCommand: execute this command after starting the
backup.
* Root: root directory of tar backup, "/" when missing.
* Include: list of pathes to include, ["."] when missing.
* Exclude: list of patterns to exclude from backup (see tar
documentation "--exclude"). When missing and Root is "/",
list ["./var/lib/guerillabackup/data"] is used.
* IgnoreBackupRaces: flag to indicate if races during backup
are acceptable, e.g. because the directories are modified,
files changed or removed. When set, such races will not
result in non-zero exit status. Off course, it would be
more sensible to deploy a snapshot based backup variant
using the Pre/PostBackupCommand functions.
* FullBackupTiming: tuple with minimum and maximum interval
between full backup invocations and modulo base and offset,
all in seconds. Without modulo invocation (all values None),
full backups will run as soon as minimum interval is exceeded.
With modulo timing, modulo trigger is ignored when below
minimum time. When gap above maximum interval, immediate
backup is started.
* IncBackupTiming: When set, incremental backups are created
to fill the time between full backups. Timings are specified
as tuple with same meaning as in FullBackupTiming parameter.
This will also trigger generation of tar file indices when
running full backups.
* FullOverrideCommand: when set, parameters Exclude, Include,
Root are ignored and exactly the given command is executed.
* IncOverrideCommand: when set, parameters Exclude, Include,
Root are ignored and exactly the given command is executed.
* KeepIndices: number of old incremental tar backup indices
to keep. With -1 keep all, otherwise keep one the given
number. Default is 0.
* Policy: If not none, include this string as handling policy
within the manifest.
* EncryptionKey: If not None, encrypt the input using the
named key. Otherwise default encryption key from global
configuration might be used."""
if not isinstance(sourceUrl, str):
raise Exception('Source URL has to be string')
guerillabackup.assertSourceUrlSpecificationConforming(sourceUrl)
self.sourceUrl = sourceUrl
if not isinstance(descriptionDict, dict):
raise Exception('Input description has to be dictionary')
self.preBackupCommandList = None
self.postBackupCommandList = None
self.backupRoot = None
self.backupIncludeList = None
self.backupExcludeList = None
self.ignoreBackupRacesFlag = False
self.fullBackupTiming = None
self.incBackupTiming = None
self.fullBackupOverrideCommand = None
self.incBackupOverrideCommand = None
self.handlingPolicyName = None
self.encryptionKeyName = None
self.keepOldIndicesCount = 0
for configKey, configValue in descriptionDict.items():
if ((configKey == 'PreBackupCommand') or
(configKey == 'PostBackupCommand') or
(configKey == 'FullOverrideCommand') or
(configKey == 'IncOverrideCommand')):
if not guerillabackup.isValueListOfType(configValue, str):
raise Exception('Parameter %s has to be list of string' % configKey)
if configKey == 'PreBackupCommand':
self.preBackupCommandList = configValue
elif configKey == 'PostBackupCommand':
self.postBackupCommandList = configValue
elif configKey == 'FullOverrideCommand':
self.fullBackupOverrideCommand = configValue
elif configKey == 'IncOverrideCommand':
self.incBackupOverrideCommand = configValue
else:
raise Exception('Logic error')
elif configKey == 'Root':
self.backupRoot = configValue
elif configKey == 'Include':
if not (isinstance(configValue, list) or
isinstance(configValue, tuple)):
raise Exception(
'Parameter %s has to be list or tuple' % configKey)
self.backupIncludeList = configValue
elif configKey == 'Exclude':
if not (isinstance(configValue, list) or
isinstance(configValue, tuple)):
raise Exception(
'Parameter %s has to be list or tuple' % configKey)
self.backupExcludeList = configValue
elif configKey == 'IgnoreBackupRaces':
self.ignoreBackupRacesFlag = configValue
elif ((configKey == 'FullBackupTiming') or
(configKey == 'IncBackupTiming')):
if (not isinstance(configValue, list)) or (len(configValue) != 4):
raise Exception(
'Parameter %s has to be list with 4 values' % configKey)
if configValue[0] is None:
raise Exception('Parameter %s minimum interval value must not be None' % configKey)
for timeValue in configValue:
if (timeValue != None) and (not isinstance(timeValue, int)):
raise Exception(
'Parameter %s contains non-number element' % configKey)
if configValue[2] != None:
if ((configValue[2] <= 0) or (configValue[3] < 0) or
(configValue[3] >= configValue[2])):
raise Exception(
'Parameter %s modulo timing values invalid' % configKey)
if configKey == 'FullBackupTiming':
self.fullBackupTiming = configValue
else: self.incBackupTiming = configValue
elif configKey == 'KeepIndices':
if not isinstance(configValue, int):
raise Exception('KeepIndices has to be integer value')
self.keepOldIndicesCount = configValue
elif configKey == 'Policy':
self.handlingPolicyName = configValue
elif configKey == 'EncryptionKey':
self.encryptionKeyName = configValue
else:
raise Exception('Unsupported parameter %s' % configKey)
if self.fullBackupTiming is None:
raise Exception('Mandatory FullBackupTiming parameter missing')
# The remaining values are not from the unit configuration but
# unit state persistency instead.
self.lastFullBackupTime = None
self.lastAnyBackupTime = None
self.lastUuidValue = None
# When not None, delay execution of any any backup for that resource
# beyond the given time. This is intended for backups that failed
# to run to avoid invocation loops. This value is not persistent
# between multiple software invocations.
self.nextRetryTime = None