def backupZODB(self):
"""
Backup the Zope database.
"""
timer = Duration().start()
self._backup.backupZodb()
self._backup.backupZodbSession()
self.log.info("Backup of ZODB database completed in %s.", readable_time(timer.elapsed().total_seconds()))
def logflush_interval(self):
"""
Return log flush interval value in readable format.
In case no value was matched, return None.
"""
ms = matcher(self.conf, self.rule(
'hbase.regionserver.optionallogflushinterval'
), 1000) # Set to default if not set in conf file.
if ms:
return readable_time(int(ms) / 1000, 2)
def logflush_interval(self):
"""
Return log flush interval value in readable format.
In case no value was matched, return None.
"""
ms = matcher(self.conf,
self.rule('hbase.regionserver.optionallogflushinterval'),
1000) # Set to default if not set in conf file.
if ms:
return readable_time(int(ms) / 1000, 2)
def backupZODB(self):
"""
Backup the Zope database.
"""
timer = Duration().start()
self._backup.backupZodb()
self._backup.backupZodbSession()
self.log.info("Backup of ZODB database completed in %s.",
readable_time(timer.elapsed().total_seconds()))
def backupZenPacks(self):
duration = Duration().start()
if self.options.backupZenPacks:
if self.options.backupZopeDb:
self._backup.backupZenPackFiles()
self._backup.backupZenPackDelegate()
self.log.info("Backup of ZenPacks completed in %s.", readable_time(duration.elapsed().total_seconds()))
else:
self.log.info('Skipping backup of ZenPack data.')
def backupZenPacks(self):
duration = Duration().start()
if self.options.backupZenPacks:
if self.options.backupZopeDb:
self._backup.backupZenPackFiles()
self._backup.backupZenPackDelegate()
self.log.info("Backup of ZenPacks completed in %s.",
readable_time(duration.elapsed().total_seconds()))
else:
self.log.info('Skipping backup of ZenPack data.')
def backupZEP(self):
'''
Backup ZEP
'''
partBeginTime = time.time()
# Setup defaults for db info
if self.options.fetchArgs:
self.log.info(
'Getting ZEP dbname, user, password, port from configuration files.'
)
self.readZEPSettings()
if self.options.saveSettings:
self.saveSettings()
if self.options.noEventsDb:
self.log.info('Doing a partial backup of the events database.')
tables = [
'config', 'event_detail_index_config', 'event_trigger',
'event_trigger_subscription', 'schema_version'
]
else:
self.log.info('Backing up the events database.')
tables = None
self.backupMySqlDb(self.options.zepdbhost,
self.options.zepdbport,
self.options.zepdbname,
self.options.zepdbuser,
'zepdbpass',
'zep.sql.gz',
tables=tables)
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Backup of events database completed in %s.",
subtotalTime)
if not self.options.noEventsDb:
zeneventserver_dir = zenPath('var', 'zeneventserver')
if self.options.noZepIndexes:
self.log.info('Not backing up event indexes.')
elif self._zepRunning():
self.log.info(
'Not backing up event indexes - it is currently running.')
elif os.path.isdir(zeneventserver_dir):
self.log.info('Backing up event indexes.')
zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'),
'w')
zepTar.add(zeneventserver_dir, 'zeneventserver')
zepTar.close()
self.log.info('Backing up event indexes completed.')
def _delayNextCheck(self):
"""
Rather than keep re-polling at the same periodicity to
determine if the device's agent is responding or not,
let this task back up in the queue.
Add a random element to it so that we don't get the
thundering herd effect.
A maximum delay is used so that there is a bound on the
length of times between checks.
"""
# If it's not responding, don't poll it so often
if self.interval != self._maxbackoffseconds:
delay = random.randint(int(self.interval / 2), self.interval) * 2
self.interval = min(self._maxbackoffseconds, self.interval + delay)
log.debug("Delaying next check for another %s", readable_time(self.interval))
def backupPerfData(self):
"""
Back up the RRD files storing performance data.
"""
duration = Duration().start()
if self.options.backupPerfData:
try:
if self._backup.backupPerf():
subtotalTime = readable_time(duration.elapsed().totalSeconds())
self.log.info("Backup of performance data completed in %s.",
subtotalTime)
except ZenBackupException as e:
self.hasCriticalErrors = self.hasCriticalErrors or e.isCritical()
self.messages.append(str(e))
else:
self.log.info('Skipping backup of performance data.')
def _delayNextCheck(self):
"""
Rather than keep re-polling at the same periodicity to
determine if the device's agent is responding or not,
let this task back up in the queue.
Add a random element to it so that we don't get the
thundering herd effect.
A maximum delay is used so that there is a bound on the
length of times between checks.
"""
# If it's not responding, don't poll it so often
if self.interval != self._maxbackoffseconds:
delay = random.randint(int(self.interval / 2), self.interval) * 2
self.interval = min(self._maxbackoffseconds, self.interval + delay)
log.debug("Delaying next check for another %s",
readable_time(self.interval))
def waitForZODBBackup(self):
"""
Wait for the ZODB backup to finish (which was kicked off from the method startBackupZODB)
"""
partBeginTime = time.time()
self.log.info("Waiting for the ZODB backup to finish...")
self._zodb_mysqldump_process.wait()
self._zodb_backup_gzip_process.wait()
self._zodb_backup_file_handler.close()
self._zodb_session_mysqldump_process.wait()
self._zodb_session_backup_gzip_process.wait()
self._zodb_session_backup_file_handler.close()
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Waited %s seconds for the ZODB backup to finish.", subtotalTime)
def waitForZODBBackup(self):
"""
Wait for the ZODB backup to finish (which was kicked off from the method startBackupZODB)
"""
partBeginTime = time.time()
self.log.info("Waiting for the ZODB backup to finish...")
self._zodb_mysqldump_process.wait()
self._zodb_backup_gzip_process.wait()
self._zodb_backup_file_handler.close()
self._zodb_session_mysqldump_process.wait()
self._zodb_session_backup_gzip_process.wait()
self._zodb_session_backup_file_handler.close()
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Waited %s seconds for the ZODB backup to finish.",
subtotalTime)
def backupPerfData(self):
"""
Back up the RRD files storing performance data.
"""
duration = Duration().start()
if self.options.backupPerfData:
try:
if self._backup.backupPerf():
subtotalTime = readable_time(
duration.elapsed().totalSeconds())
self.log.info(
"Backup of performance data completed in %s.",
subtotalTime)
except ZenBackupException as e:
self.hasCriticalErrors = self.hasCriticalErrors or e.isCritical(
)
self.messages.append(str(e))
else:
self.log.info('Skipping backup of performance data.')
def checkDuration(self, dp, values):
'Check the value for point thresholds'
log.debug("Checking %s %s against interval %s",
dp, values, self.interval)
if not values:
return []
values = map(float, values)
avg_value = sum(values)/len(values)
thresh = None
if self.maximum is not None and avg_value > self.maximum:
thresh = self.maximum
how = 'above'
if self.minimum is not None and avg_value < self.minimum:
thresh = self.minimum
how = 'below'
if thresh is not None:
severity = self.severity
count = self.incrementCount(dp)
if self.escalateCount and count >= self.escalateCount:
severity = min(severity + 1, 5)
summary = 'Threshold of %s: in the last %s average value was %s %s.' % (
self.name(), readable_time(self.interval), how, thresh)
return [dict(device=self.context().deviceName,
summary=summary,
eventKey=self.id,
eventClass=self.eventClass,
component=self.context().componentName,
severity=severity)]
else:
count = self.getCount(dp)
if count is None or count > 0:
summary = 'Threshold of %s restored: current value: %.2f' % (
self.name(), avg_value)
self.resetCount(dp)
return [dict(device=self.context().deviceName,
summary=summary,
eventKey=self.id,
eventClass=self.eventClass,
component=self.context().componentName,
severity=Event.Clear)]
return []
def backupZEP(self):
'''
Backup ZEP
'''
partBeginTime = time.time()
# Setup defaults for db info
if self.options.fetchArgs:
self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
self.readZEPSettings()
if self.options.saveSettings:
self.saveSettings()
if self.options.noEventsDb:
self.log.info('Doing a partial backup of the events database.')
tables=['config','event_detail_index_config','event_trigger','event_trigger_subscription', 'schema_version']
else:
self.log.info('Backing up the events database.')
tables = None
self.backupMySqlDb(self.options.zepdbhost, self.options.zepdbport,
self.options.zepdbname, self.options.zepdbuser,
'zepdbpass', 'zep.sql.gz', tables=tables)
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Backup of events database completed in %s.", subtotalTime)
if not self.options.noEventsDb:
zeneventserver_dir = zenPath('var', 'zeneventserver')
if self.options.noZepIndexes:
self.log.info('Not backing up event indexes.')
elif self._zepRunning():
self.log.info('Not backing up event indexes - it is currently running.')
elif os.path.isdir(zeneventserver_dir):
self.log.info('Backing up event indexes.')
zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'), 'w')
zepTar.add(zeneventserver_dir, 'zeneventserver')
zepTar.close()
self.log.info('Backing up event indexes completed.')
def backupZep(self):
# Do a full backup of zep if option set, otherwise only back
# up a small subset of tables to capture the event triggers.
try:
timer = Duration().start()
if self.options.backupEventsDb:
self._backup.backupZepDb()
if self.options.backupZepIndexes:
self._backup.backupZepIndexFiles()
elif self.options.backupZopeDb:
#This backups the triggers, actions etc, that are need to be consistent between zep and zodb
self._backup.backupZepDb(full=False)
else:
self.log.info('Skipping backup of the events database.')
self.log.info("Backup of events database completed in %s.", readable_time(timer.elapsed().total_seconds()))
except ZenBackupException as e:
self.hasCriticalErrors = self.hasCriticalErrors or e.isCritical()
self.messages.append(str(e))
def backupZep(self):
# Do a full backup of zep if option set, otherwise only back
# up a small subset of tables to capture the event triggers.
try:
timer = Duration().start()
if self.options.backupEventsDb:
self._backup.backupZepDb()
if self.options.backupZepIndexes:
self._backup.backupZepIndexFiles()
elif self.options.backupZopeDb:
#This backups the triggers, actions etc, that are need to be consistent between zep and zodb
self._backup.backupZepDb(full=False)
else:
self.log.info('Skipping backup of the events database.')
self.log.info("Backup of events database completed in %s.",
readable_time(timer.elapsed().total_seconds()))
except ZenBackupException as e:
self.hasCriticalErrors = self.hasCriticalErrors or e.isCritical()
self.messages.append(str(e))
def backupPerfData(self):
"""
Back up the RRD files storing performance data.
"""
perfDir = zenPath('perf')
if not os.path.isdir(perfDir):
self.log.warning('%s does not exist, skipping.', perfDir)
return
partBeginTime = time.time()
self.log.info('Backing up performance data (RRDs).')
tarFile = os.path.join(self.tempDir, 'perf.tar')
#will change dir to ZENHOME so that tar dir structure is relative
cmd = ['tar', 'chfC', tarFile, zenPath(), 'perf']
(output, warnings, returncode) = self.runCommand(cmd)
if returncode:
raise ZenBackupException("Performance Data backup failed.", True)
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Backup of performance data completed in %s.",
subtotalTime)
def backupPerfData(self):
"""
Back up the RRD files storing performance data.
"""
perfDir = zenPath('perf')
if not os.path.isdir(perfDir):
self.log.warning('%s does not exist, skipping.', perfDir)
return
partBeginTime = time.time()
self.log.info('Backing up performance data (RRDs).')
tarFile = os.path.join(self.tempDir, 'perf.tar')
#will change dir to ZENHOME so that tar dir structure is relative
cmd = ['tar', 'chfC', tarFile, zenPath(), 'perf']
(output, warnings, returncode) = self.runCommand(cmd)
if returncode:
raise ZenBackupException("Performance Data backup failed.", True)
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Backup of performance data completed in %s.",
subtotalTime )
def makeBackup(self):
'''
Create a backup of the data and configuration for a Zenoss install.
'''
hasCriticalErrors = False
messages = []
backupBeginTime = time.time()
# Create temp backup dir
self.rootTempDir = self.getTempDir()
self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
self.log.debug("Use %s as a staging directory for the backup", self.tempDir)
os.mkdir(self.tempDir, 0750)
if self.options.collector:
self.options.noEventsDb = True
self.options.noZopeDb = True
self.options.noZepIndexes = True
self.options.noZenPacks = True
# Do a full backup of zep if noEventsDb is false, otherwise only back
# up a small subset of tables to capture the event triggers.
if not self.options.noEventsDb or not self.options.noZopeDb:
try:
self.backupZEP()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
else:
self.log.info('Skipping backup of the events database.')
# Copy /etc to backup dir (except for sockets)
self.log.info('Backing up config files.')
etcTar = tarfile.open(os.path.join(self.tempDir, 'etc.tar'), 'w')
etcTar.dereference = True
etcTar.add(zenPath('etc'), 'etc')
etcTar.close()
self.log.info("Backup of config files completed.")
if self.options.noZopeDb:
self.log.info('Skipping backup of ZODB.')
else:
notify(PreBackupEvent(self))
self.startBackupZODB()
if self.options.noZenPacks:
self.log.info('Skipping backup of ZenPack data.')
else:
partBeginTime = time.time()
self.backupZenPacks()
self.backupZenPackContents()
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Backup of ZenPacks completed in %s.", subtotalTime)
notify(PostBackupEvent(self))
if not self.options.noZopeDb:
self.waitForZODBBackup()
if self.options.noPerfData:
self.log.info('Skipping backup of performance data.')
else:
try:
self.backupPerfData()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
# tar, gzip and send to outfile
try:
outfile = self.packageStagingBackups()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
try:
self.cleanupTempDir()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
if not hasCriticalErrors:
backupEndTime = time.time()
totalBackupTime = readable_time(backupEndTime - backupBeginTime)
if len(messages) == 0:
self.log.info('Backup completed successfully in %s.', totalBackupTime)
else:
self.log.info('Backup completed successfully in %s, but the following errors occurred:', totalBackupTime)
for msg in messages:
self.log.error(msg)
return 0
else:
for msg in messages:
self.log.critical(msg)
return -1
def makeBackup(self):
'''
Create a backup of the data and configuration for a Zenoss install.
'''
hasCriticalErrors = False
messages = []
backupBeginTime = time.time()
# Create temp backup dir
self.rootTempDir = self.getTempDir()
self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
self.log.debug("Use %s as a staging directory for the backup",
self.tempDir)
os.mkdir(self.tempDir, 0750)
if self.options.collector:
self.options.noEventsDb = True
self.options.noZopeDb = True
self.options.noZepIndexes = True
self.options.noZenPacks = True
# Do a full backup of zep if noEventsDb is false, otherwise only back
# up a small subset of tables to capture the event triggers.
if not self.options.noEventsDb or not self.options.noZopeDb:
try:
self.backupZEP()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
else:
self.log.info('Skipping backup of the events database.')
# Copy /etc to backup dir (except for sockets)
self.log.info('Backing up config files.')
etcTar = tarfile.open(os.path.join(self.tempDir, 'etc.tar'), 'w')
etcTar.dereference = True
etcTar.add(zenPath('etc'), 'etc')
etcTar.close()
self.log.info("Backup of config files completed.")
if self.options.noZopeDb:
self.log.info('Skipping backup of ZODB.')
else:
notify(PreBackupEvent(self))
self.startBackupZODB()
if self.options.noZenPacks:
self.log.info('Skipping backup of ZenPack data.')
else:
partBeginTime = time.time()
self.backupZenPacks()
self.backupZenPackContents()
partEndTime = time.time()
subtotalTime = readable_time(partEndTime - partBeginTime)
self.log.info("Backup of ZenPacks completed in %s.", subtotalTime)
notify(PostBackupEvent(self))
if not self.options.noZopeDb:
self.waitForZODBBackup()
if self.options.noPerfData:
self.log.info('Skipping backup of performance data.')
else:
try:
self.backupPerfData()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
# tar, gzip and send to outfile
try:
outfile = self.packageStagingBackups()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
try:
self.cleanupTempDir()
except ZenBackupException as e:
hasCriticalErrors = hasCriticalErrors or e.isCritical()
messages.append(str(e))
if not hasCriticalErrors:
backupEndTime = time.time()
totalBackupTime = readable_time(backupEndTime - backupBeginTime)
if len(messages) == 0:
self.log.info('Backup completed successfully in %s.',
totalBackupTime)
else:
self.log.info(
'Backup completed successfully in %s, but the following errors occurred:',
totalBackupTime)
for msg in messages:
self.log.error(msg)
return 0
else:
for msg in messages:
self.log.critical(msg)
return -1
def doBackup(self):
"""
Create a backup of the data and configuration for a Zenoss install.
"""
backupBeginTime = time.time()
# Create temp backup dir
self.rootTempDir = self.getTempDir()
self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
self.log.debug("Use %s as a staging directory for the backup", self.tempDir)
os.mkdir(self.tempDir, 0750)
if self.options.fetchArgs:
self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
self.readZEPSettings()
if self.options.collector:
self.options.backupEventsDb = False
self.options.backupZopeDb = False
self.options.backupZepIndexes = False
self.options.backupZenPacks = False
# now that settings are read, initialize backup impl
self._backup = self._backupFactory.getBackupImpl(self.tempDir, self.options)
if self.options.saveSettings:
self._backup.backupSettings()
self.backupZep()
# Copy /etc to backup dir (except for sockets)
self._backup.backupConfFiles()
if self.options.backupZopeDb:
notify(PreBackupEvent(self))
self.backupZODB()
else:
self.log.info('Skipping backup of ZODB.')
self.backupZenPacks()
notify(PostBackupEvent(self))
self.backupPerfData()
# tar, gzip and send to outfile
self.packageStagingBackups()
self.cleanupTempDir()
if not self.hasCriticalErrors:
backupEndTime = time.time()
totalBackupTime = readable_time(backupEndTime - backupBeginTime)
if len(self.messages) == 0:
self.log.info('Backup completed successfully in %s.',
totalBackupTime)
else:
self.log.info('Backup completed successfully in %s, but the \
following errors occurred:', totalBackupTime)
for msg in self.messages:
self.log.error(msg)
return 0
else:
for msg in self.messages:
self.log.critical(msg)
return -1
def doBackup(self):
"""
Create a backup of the data and configuration for a Zenoss install.
"""
backupBeginTime = time.time()
# Create temp backup dir
self.rootTempDir = self.getTempDir()
self.tempDir = os.path.join(self.rootTempDir, BACKUP_DIR)
self.log.debug("Use %s as a staging directory for the backup",
self.tempDir)
os.mkdir(self.tempDir, 0750)
if self.options.fetchArgs:
self.log.info(
'Getting ZEP dbname, user, password, port from configuration files.'
)
self.readZEPSettings()
if self.options.collector:
self.options.backupEventsDb = False
self.options.backupZopeDb = False
self.options.backupZepIndexes = False
self.options.backupZenPacks = False
# now that settings are read, initialize backup impl
self._backup = self._backupFactory.getBackupImpl(
self.tempDir, self.options)
if self.options.saveSettings:
self._backup.backupSettings()
self.backupZep()
# Copy /etc to backup dir (except for sockets)
self._backup.backupConfFiles()
if self.options.backupZopeDb:
notify(PreBackupEvent(self))
self.backupZODB()
else:
self.log.info('Skipping backup of ZODB.')
self.backupZenPacks()
notify(PostBackupEvent(self))
self.backupPerfData()
# tar, gzip and send to outfile
self.packageStagingBackups()
self.cleanupTempDir()
if not self.hasCriticalErrors:
backupEndTime = time.time()
totalBackupTime = readable_time(backupEndTime - backupBeginTime)
if len(self.messages) == 0:
self.log.info('Backup completed successfully in %s.',
totalBackupTime)
else:
self.log.info(
'Backup completed successfully in %s, but the \
following errors occurred:', totalBackupTime)
for msg in self.messages:
self.log.error(msg)
return 0
else:
for msg in self.messages:
self.log.critical(msg)
return -1
