def createRRDFile( self, dataPointName, rrdCreateCommand ):
"""
Create an RRD file if it does not exist or if the step value has changed.
Returns the basename of the rrdFile, suitable for checking thresholds.
"""
self.log.debug( 'Checking RRD File for %s' % dataPointName )
if not self.daemonName:
return
directory = zenPath( 'perf', 'Daemons', self.monitor )
if not os.path.exists( directory ):
self.log.debug( 'Creating directory: %s' % directory )
os.makedirs( directory )
fileName = '%s_%s.rrd' % ( self.daemonName, dataPointName )
filePath = zenPath( directory, fileName )
step = rrdCreateCommand[rrdCreateCommand.index('--step')+1]
if not os.path.exists( filePath ) or \
self._hasStepTimeChanged( filePath, step ):
import getpass
self.log.debug( 'Creating RRD file %s as user %s with options %s' %
( filePath, getpass.getuser(), rrdCreateCommand ) )
rrdtool.create( filePath, *rrdCreateCommand )
else:
self.log.debug( 'RRD file already exists' )
def DoEasyUninstall(name):
"""
Execute the easy_install command to unlink the given egg.
What this is really doing is switching the egg to be in
multiple-version mode, however this is the first step in deleting
an egg as described here:
http://peak.telecommunity.com/DevCenter/EasyInstall#uninstalling-packages
"""
from setuptools.command import easy_install
args = ['--site-dirs', zenPath('ZenPacks'),
'-d', zenPath('ZenPacks'),
#'--record', tempPath,
'--quiet',
'-m',
name]
#Try to run the easy install command. If this fails with an attribute
#error, then we know that easy_install doesn't know about this ZenPack and
#we can continue normally
try:
easy_install.main(args)
except AttributeError:
log.info("%s not found by easy_install. Continuing to remove." % name)
pass
def generate_router_db(self):
db_files = {}
customdb_files = {}
for dev in self._rancid:
if not (dev['id'] and dev['group'] and dev['deviceType']): continue
groupDir = zenPath('rancid', 'var', dev['group'])
routerDbFile = os.path.join(groupDir, "router.db")
if os.path.exists(groupDir):
if not routerDbFile in db_files:
db_files[routerDbFile] = []
# store router.db in lower case
db_files[routerDbFile].append(":".join((dev['id'], dev['deviceType'], "up")))
for file in db_files:
out = open(file, 'w')
for router in db_files[file]:
out.write(str(router).lower() + "\n")
# check if a custom router.db file exists in rancid/etc/customdb
customDbFile = re.sub(zenPath('rancid', 'var'), zenPath('rancid', 'etc', 'customdb'), file)
if os.path.exists(customDbFile):
with open(customDbFile) as f:
out.write(str(f.readline()))
out.close()
def _findEggs(self, zenpackID, zenpackVersion):
# if the version has a dash, replace with an underscore
# and format it
zenpackVersion = parse_version(zenpackVersion.replace("-", "_", 1))
eggs = []
for dirpath in zenPath(".ZenPacks"), zenPath("packs"):
for f in os.listdir(dirpath):
# verify the .egg extension and strip it from the egg name
base, ext = os.path.splitext(f)
if ext != ".egg":
continue
# make sure this is the standard egg name format
match = EGG_NAME(base)
if not match:
continue
# extrapolate the values of the name and version
# NOTE: also has groupings for 'pyver' and 'plat' if needed for
# later.
name, version = match.group('name', 'ver')
# formatting version to a standart
version = parse_version(version)
if name == zenpackID and (not zenpackVersion or version == zenpackVersion):
eggs.append(os.path.join(dirpath, f))
# no point in checking the other dirpaths if an egg has been found
if len(eggs) > 0:
break
return eggs
def restoreZEP(self):
'''
Restore ZEP DB and indexes
'''
zepSql = self.getSqlFile('zep.sql')
if not zepSql:
self.msg('This backup does not contain a ZEP database backup.')
return
# Setup defaults for db info
if self.options.fetchArgs:
self.log.info('Getting ZEP dbname, user, password, port from configuration files.')
self.readZEPSettings()
self.msg('Restoring ZEP database.')
self.restoreMySqlDb(self.options.zepdbhost, self.options.zepdbport,
self.options.zepdbname, self.options.zepdbuser,
self.getPassArg('zepdbpass'), zepSql)
self.msg('ZEP database restored.')
# Remove any current indexes on the system
index_dir = zenPath('var', 'zeneventserver', 'index')
if os.path.isdir(index_dir):
import shutil
self.msg('Removing existing ZEP indexes.')
shutil.rmtree(index_dir)
index_tar = os.path.join(self.tempDir, 'zep.tar')
if os.path.isfile(index_tar):
self.msg('Restoring ZEP indexes.')
zepTar = tarfile.open(os.path.join(self.tempDir, 'zep.tar'))
zepTar.extractall(zenPath('var'))
self.msg('ZEP indexes restored.')
else:
self.msg('ZEP indexes not found in backup file - will be recreated from database.')
def installFile(self, relativePath, relativeDestPath=None,
overwriteIfExists=True, symLink=False):
"""
Install a file from this zenpack to ZENHOME upon installation.
By default, relativePath is for the zenpack and its ZENHOME destination.
Returns True if installed, False/Exception otherwise.
Example: self.installFile('etc/myzenpack.data')
"""
srcPath = self.path(relativePath)
destPath = zenPath(relativePath) if relativeDestPath is None \
else zenPath(relativeDestPath)
if not overwriteIfExists and os.path.lexists(destPath):
return False
if not os.path.exists(srcPath):
raise ZenPackException('Missing source file %s' % srcPath)
if os.path.lexists(destPath):
os.remove(destPath)
# If the file is open for edit in Unix then it'll still exist
# until it's closed, which means we can't overwrite it.
if os.path.lexists(destPath):
raise ZenPackException('Failed to remove file %s' % destPath)
if symLink:
os.symlink(srcPath, destPath)
else:
shutil.copy2(srcPath, destPath)
if not os.path.lexists(destPath):
raise ZenPackException('Failed to write file %s' % destPath)
return True
def copyDir(self, srcDir):
"""Copy an unzipped zenpack to the appropriate location.
Return the name.
"""
# Normalize srcDir to not end with slash
if srcDir.endswith('/'):
srcDir = srcDir[:-1]
if not os.path.isdir(srcDir):
self.stop('Specified directory does not appear to exist: %s' %
srcDir)
# Determine name of pack and it's destination directory
packName = os.path.split(srcDir)[1]
root = zenPath('Products', packName)
# Continue without copying if the srcDir is already in Products
if os.path.exists(root) and os.path.samefile(root, srcDir):
self.log.debug('Directory already in %s, not copying.',
zenPath('Products'))
return packName
# Copy the source dir over to Products
self.log.debug('Copying %s' % packName)
result = os.system('cp -r %s %s' % (srcDir, zenPath('Products')))
if result == -1:
self.stop('Error copying %s to %s' % (srcDir, zenPath('Products')))
return packName
def linkDir(self, srcDir):
"""Symlink the srcDir into Products
Return the name.
"""
# Normalize srcDir to not end with slash
if srcDir.endswith('/'):
srcDir = srcDir[:-1]
# Need absolute path for links
srcDir = os.path.abspath(srcDir)
if not os.path.isdir(srcDir):
self.stop('Specified directory does not appear to exist: %s' %
srcDir)
# Determine name of pack and it's destination directory
packName = os.path.split(srcDir)[1]
root = zenPath('Products', packName)
# Continue without copying if the srcDir is already in Products
if os.path.exists(root) and os.path.samefile(root, srcDir):
self.log.debug('Directory already in %s, not copying.',
zenPath('Products'))
return packName
targetdir = zenPath("Products", packName)
cmd = 'test -d %s && rm -rf %s' % (targetdir, targetdir)
os.system(cmd)
cmd = 'ln -s %s %s' % (srcDir, zenPath("Products"))
os.system(cmd)
return packName
def restorePerfData(self):
cmd = 'rm -rf %s' % os.path.join(zenPath(), 'perf')
if os.system(cmd): return -1
self.msg('Restoring performance data.')
cmd = 'tar Cxf %s %s' % (
zenPath(),
os.path.join(self.tempDir, 'perf.tar'))
if os.system(cmd): return -1
def generateCustomDbPath(self):
path = zenPath('rancid', 'etc','customdb')
if not os.path.isdir(path):
os.mkdir(path, 0750)
out = open(zenPath('rancid', 'etc','customdb', 'readme'), 'w')
out.write("# create your own custom router.db files for devices which are not managed in Zenoss\n")
out.write("# create the same folder structure as exists in rancid/var and put your own router.db file inside\n")
out.close()
def _loadConfig(self, infix=None):
if infix:
self.config_filename = zenPath('etc/zeneventmigrate-%s.conf' % infix)
elif self.options.startoffset:
self.config_filename = zenPath('etc/zeneventmigrate-%d.conf' % self.options.startoffset)
self.config = ConfigParser()
self.config.read(self.config_filename)
if not self.config.has_section(self.config_section):
self.config.add_section(self.config_section)
def FetchZenPack(dmd, zenPackName):
"""
Use easy_install to retrieve the given zenpack and any dependencies.
easy_install will install the eggs, but does not install them into
Zenoss as ZenPacks.
Return a list of distributions just installed that appear to be
ZenPacks.
NB: This should be refactored. It shares most of its code with
DoEasyInstall()
"""
from setuptools.command import easy_install
# Make sure $ZENHOME/ZenPacks exists
CreateZenPacksDir()
# Create the proper package index URL.
index_url = "%s/%s/%s/" % (ZENPACKS_BASE_URL, dmd.uuid, ZVersion.VERSION)
# Create temp file for easy_install to write results to
_, tempPath = tempfile.mkstemp(prefix="zenpackcmd-easyinstall")
# eggPaths is a set of paths to eggs that were installed. We need to
# add them to the current workingset so we can discover their
# entry points.
eggPaths = set()
try:
# Execute the easy_install
args = [
"--site-dirs",
zenPath("ZenPacks"),
"-d",
zenPath("ZenPacks"),
"-i",
index_url,
"--allow-hosts",
urlparse(index_url).hostname,
"--record",
tempPath,
"--quiet",
zenPackName,
]
easy_install.main(args)
# Collect the paths for eggs that were installed
f = open(tempPath, "r")
marker = ".egg/"
markerOffset = len(marker) - 1
for l in f.readlines():
i = l.find(marker)
if i > 0:
eggPaths.add(l[: i + markerOffset])
finally:
os.remove(tempPath)
# Add any installed eggs to the current working set
zpDists = []
for path in eggPaths:
zpDists += AddDistToWorkingSet(path)
return zpDists
def _restore(self, zenpackID, zenpackVersion, filesOnly):
# if the version has a dash, replace with an underscore
zenpackVersion = zenpackVersion.replace("-", "_", 1)
# look for the egg
eggs = []
for dirpath in zenPath(".ZenPacks"), zenPath("packs"):
for f in os.listdir(dirpath):
# verify the .egg extension and strip it from the egg name
base, ext = os.path.splitext(f)
if ext != ".egg":
continue
# make sure this is the standard egg name format
match = EGG_NAME(base)
if not match:
continue
# extrapolate the values of the name and version
# NOTE: also has groupings for 'pyver' and 'plat' if needed for
# later.
name, version = match.group('name', 'ver')
if name == zenpackID and (not zenpackVersion or version == zenpackVersion):
eggs.append(os.path.join(dirpath, f))
# no point in checking the other dirpaths if an egg has been found
if len(eggs) > 0:
break
if len(eggs) == 0:
self.log.info("Could not find install candidate for %s (%s)", zenpackID, zenpackVersion)
return
elif len(eggs) > 1:
eggpaths = ", ".join(eggs)
self.log.error("Found more than one install candidate for %s (%s): %s [skipping]", zenpackID, zenpackVersion, eggpaths)
return
candidate = eggs[0]
self.log.info("Loading candidate %s", candidate)
if candidate.lower().endswith(".egg"):
try:
shutil.copy(candidate, tempfile.gettempdir())
cmd = ["zenpack"]
if filesOnly:
cmd.append("--files-only")
cmd.extend(["--install", os.path.join(tempfile.gettempdir(), os.path.basename(candidate))])
try:
with open(os.devnull, 'w') as fnull:
# the first time fixes the easy-install path
subprocess.check_call(cmd, stdout=fnull, stderr=fnull)
except Exception:
pass
# the second time runs the loaders
subprocess.check_call(cmd)
finally:
try:
os.remove(os.path.join(tempfile.gettempdir(), os.path.basename(candidate)))
except Exception:
pass
else:
self.log.warning("non-egg zenpacks can not currently be restored automatically: %s", candidate)
def buildOptions(self):
"""
Command-line options
"""
super(ZenMib, self).buildOptions()
self.parser.add_option(
'--mibsdir', dest='mibsdir', default=zenPath('share/mibs/site'),
help="Directory of input MIB files [default: %default]"
)
self.parser.add_option(
'--mibdepsdir', dest='mibdepsdir', default=zenPath('share/mibs'),
help="Directory of input MIB files [default: %default]"
)
self.parser.add_option(
'--path', dest='path', default="",
help="Path to load MIB into the DMD [default: %default]"
)
self.parser.add_option(
'--nocommit', dest='nocommit', action='store_true', default=False,
help="Don't commit the MIB to the DMD after loading"
)
self.parser.add_option(
'--keeppythoncode', dest='keeppythoncode',
action='store_true', default=False,
help="Save the generated Python code"
)
self.parser.add_option(
'--pythoncodedir', dest='pythoncodedir',
default=tempfile.gettempdir() + "/mib_pythoncode/",
help="This is the directory where the converted MIB will be "
"output [default: %default]"
)
self.parser.add_option(
'--downloaddir', dest='downloaddir',
default=tempfile.gettempdir() + "/mib_downloads/",
help="This is the directory where the MIB will be downloaded "
"[default: %default]"
)
self.parser.add_option(
'--extractdir', dest='extractdir',
default=tempfile.gettempdir() + "/mib_extract/",
help="This is the directory where unzipped MIB files will be "
"stored [default: %default]"
)
self.parser.add_option(
'--evalSavedPython', dest='evalSavedPython',
action='append', default=[],
help="Execute the Python code previously generated and saved"
)
self.parser.add_option(
'--removemiddlezeros', dest='removemiddlezeros',
action='store_true', default=False,
help="Remove zeros found in the middle of the OID"
)
def cutover(self, dmd):
# Write extra path to sitecustomize.py
import sys
if zenPath('ZenPacks') in sys.path: return
path = zenPath('lib', 'python', 'sitecustomize.py')
f = open(path, 'r')
t = f.read()
f.close()
f = open(path, 'w')
f.write(t)
f.write('\nimport os, os.path, site; ')
f.write("site.addsitedir(os.path.join(os.getenv('ZENHOME'), 'ZenPacks'))\n")
f.close()
def CreateZenPack(zpId, prevZenPackName='', devDir=None):
"""
Create the zenpack in the filesystem.
The zenpack is not installed in Zenoss, it is simply created in
the $ZENHOME/ZenPacks directory. Usually this should be followed
with a "zenpack install" call.
zpId should already be valid, scrubbed value.
prevZenPackName is written to PREV_ZENPACK_NAME in setup.py.
"""
parts = zpId.split('.')
# Copy template to $ZENHOME/ZenPacks
srcDir = zenPath('Products', 'ZenModel', 'ZenPackTemplate')
if not devDir:
devDir = zenPath('ZenPacks')
if not os.path.exists(devDir):
os.mkdir(devDir, 0750)
destDir = os.path.join(devDir, zpId)
shutil.copytree(srcDir, destDir, symlinks=False)
os.system('find %s -name .svn | xargs rm -rf' % destDir)
# Write setup.py
packages = []
for i in range(len(parts)):
packages.append('.'.join(parts[:i+1]))
mapping = dict(
NAME = zpId,
VERSION = '1.0.0',
AUTHOR = '',
LICENSE = '',
NAMESPACE_PACKAGES = packages[:-1],
PACKAGES = packages,
INSTALL_REQUIRES = [],
COMPAT_ZENOSS_VERS = '',
PREV_ZENPACK_NAME = prevZenPackName,
)
WriteSetup(os.path.join(destDir, 'setup.py'), mapping)
# Create subdirectories
base = destDir
for part in parts[:-1]:
base = os.path.join(base, part)
os.mkdir(base)
f = open(os.path.join(base, '__init__.py'), 'w')
f.write("__import__('pkg_resources').declare_namespace(__name__)\n")
f.close()
base = os.path.join(base, parts[-1])
shutil.move(os.path.join(destDir, 'CONTENT'), base)
audit('Shell.ZenPack.Create', zpId)
return destDir
def _restore(self, zpId, version, filesOnly):
# glob for backup
# This is meant to handle standard zenpack naming convention - for exmaple,
# ZenPacks.zenoss.OpenStack-1.2.4dev19_5159496-py2.7.egg
backupDirs = (zenPath(".ZenPacks"), zenPath("packs"))
dashIndex = version.find('-')
# If the version has a dash in it, replace it with an underscore
if dashIndex != -1:
version = list(version)
version[dashIndex] = '_'
version = ''.join(version)
patterns = [backupDir + "/%s-%s-*" % (zpId, version) for backupDir in backupDirs]
# Look through potential .egg locations, breaking out once we find one
# (AKA prefer the first location)
for pattern in patterns:
self.log.info("looking for %s", pattern)
candidates = glob.glob(pattern)
if len(candidates) > 0:
break
if len(candidates) == 0:
self.log.info("could not find install candidate for %s %s", zpId, version)
return
if len(candidates) > 1:
self.log.error("Found more than one install candidate for %s %s (%s), skipping",
zpId, version, ", ".join(candidates))
return
# Make the code below this easier to read
candidate = candidates[0]
if candidate.lower().endswith(".egg"):
try:
shutil.copy(candidate, tempfile.gettempdir())
cmd = ["zenpack"]
if filesOnly:
cmd.append("--files-only")
cmd.extend(["--install", os.path.join(tempfile.gettempdir(), os.path.basename(candidate))])
try:
with open(os.devnull, 'w') as fnull:
# the first time fixes the easy-install path
subprocess.check_call(cmd, stdout=fnull, stderr=fnull)
except Exception:
pass
# the second time runs the loaders
subprocess.check_call(cmd)
finally:
try:
os.remove(os.path.join(tempfile.gettempdir(), os.path.basename(candidate)))
except Exception:
pass
else:
self.log.warning("non-egg zenpacks can not currently be restored automatically: %s", candidate)
def restoreEtcFiles(self):
self.msg('Restoring config files.')
cmd = 'cp -p %s %s' % (os.path.join(zenPath('etc'), 'global.conf'), self.tempDir)
if os.system(cmd): return -1
cmd = 'tar Cxf %s %s' % (
zenPath(),
os.path.join(self.tempDir, 'etc.tar')
)
if os.system(cmd): return -1
if not os.path.exists(os.path.join(zenPath('etc'), 'global.conf')):
self.msg('Restoring default global.conf')
cmd = 'mv %s %s' % (os.path.join(self.tempDir, 'global.conf'), zenPath('etc'))
if os.system(cmd): return -1
def DoEasyInstall(eggPath):
"""
Use easy_install to install an egg from the filesystem.
easy_install will install the egg, but does not install it into
Zenoss as ZenPacks.
Returns a list of distributions that were installed that appear
to be ZenPacks.
"""
from setuptools.command import easy_install
# Make sure $ZENHOME/ZenPacks exists
CreateZenPacksDir()
# Create temp file for easy_install to write results to
_, tempPath = tempfile.mkstemp(prefix="zenpackcmd-easyinstall")
# eggPaths is a set of paths to eggs that were installed. We need to
# add them to the current workingset so we can discover their
# entry points.
eggPaths = set()
try:
# Execute the easy_install
args = [
"--site-dirs",
zenPath("ZenPacks"),
"-d",
zenPath("ZenPacks"),
"--allow-hosts",
"None",
"--record",
tempPath,
"--quiet",
eggPath,
]
easy_install.main(args)
# Collect the paths for eggs that were installed
f = open(tempPath, "r")
marker = ".egg/"
markerOffset = len(marker) - 1
for l in f.readlines():
i = l.find(marker)
if i > 0:
eggPaths.add(l[: i + markerOffset])
finally:
os.remove(tempPath)
# Add any installed eggs to the current working set
zpDists = []
for path in eggPaths:
zpDists += AddDistToWorkingSet(path)
return zpDists
def manage_deleteBackups(self, fileNames=(), REQUEST=None):
"""
Delete the specified files from $ZENHOME/backups
"""
backupsDir = zenPath('backups')
removed = []
if os.path.isdir(backupsDir):
for dirPath, dirNames, dirFileNames in os.walk(backupsDir):
dirNames[:] = []
for fileName in fileNames:
if fileName in dirFileNames:
toRemove = os.path.join(dirPath, fileName)
res = os.remove(toRemove)
if not res:
removed.append(toRemove)
if REQUEST:
audit('UI.Backup.Delete', files=removed)
messaging.IMessageSender(self).sendToBrowser(
'Backups Deleted',
'%s backup files have been deleted.' % len(removed)
)
else:
if REQUEST:
messaging.IMessageSender(self).sendToBrowser(
'Backup Directory Missing',
'Unable to find $ZENHOME/backups.',
messaging.WARNING
)
if REQUEST:
return self.callZenScreen(REQUEST)
def _getCollectorFiles(self, collector):
def parseOutput(output):
files = set()
for line in (l.strip() for l in output.split("\n") if l):
files.add(line)
return files
log.debug("Checking collector %s (%s) for RRD files", collector.id, collector.hostname)
allCmd = "find %s -name *.rrd" % zenPath("perf", "Devices")
staleCmd = "%s -mtime -%s" % (allCmd, self.options.age)
if collector.hostname == "localhost":
allOutput = Popen([allCmd], shell=True, stdout=PIPE).communicate()[0]
staleOutput = Popen([staleCmd], shell=True, stdout=PIPE).communicate()[0]
else:
# Quick check to see if we can SSH to the collector.
p1 = Popen(["echo", "0"], stdout=PIPE)
p2 = Popen(["nc", "-w", "4", collector.hostname, "22"], stdin=p1.stdout, stdout=PIPE, stderr=PIPE)
if os.waitpid(p2.pid, 0)[1] != 0:
log.warn("Unable to SSH to collector %s (%s)", collector.id, collector.hostname)
return
allOutput = Popen(["ssh", collector.hostname, allCmd], stdout=PIPE).communicate()[0]
staleOutput = Popen(["ssh", collector.hostname, staleCmd], stdout=PIPE).communicate()[0]
collector.allFiles = parseOutput(allOutput)
collector.staleFiles = parseOutput(staleOutput)
def _readFilters(self):
fileName = self._daemon.options.trapFilterFile
if fileName:
path = zenPath('etc', fileName)
if os.path.exists(path):
with open(path) as filterDefinitionFile:
lineNumber = 0
for line in filterDefinitionFile:
lineNumber += 1
if line.startswith('#'):
continue
# skip blank lines
line = line.strip()
if not line:
continue;
errorMessage = self._parseFilterDefinition(line, lineNumber)
if errorMessage:
errorMessage = "Failed to parse filter definition file %s at line %d: %s" % (format(path), lineNumber, errorMessage)
raise TrapFilterError(errorMessage)
self._filtersDefined = 0 != (len(self._v1Traps) + len(self._v1Filters) + len(self._v2Filters))
if self._filtersDefined:
log.info("Finished reading filter definition file %s", format(path))
else:
log.warn("No zentrap filters found in %s", format(path))
else:
errorMessage = "Could find filter definition file %s" % format(path)
raise TrapFilterError(errorMessage)
def cutover(self, dmd):
procs = zenPath('Products', 'ZenEvents', 'db', 'zenprocs.sql')
os.system('cat %s | mysql -u%s -p%s %s' % (
procs,
dmd.ZenEventManager.username,
dmd.ZenEventManager.password,
dmd.ZenEventManager.database))
def remove(self, app, leaveObjects=False):
if not leaveObjects:
self.unregister_devtype(app.zport.dmd)
# remove kerberos.so file from python path
kerbdst = os.path.join(zenPath('lib', 'python'), 'kerberos.so')
kerbconfig = os.path.join(os.environ['ZENHOME'], 'var', 'krb5')
userenvironconfig = '{0}/.bashrc'.format(os.environ['HOME'])
try:
os.remove(kerbdst)
os.remove(kerbconfig)
# Remove export for KRB5_CONFIG from bashrc
bashfile = open(userenvironconfig, 'r')
content = bashfile.read()
bashfile.close()
content = re.sub(r'# Following value required for Windows ZenPack\n?',
'',
content)
content = re.sub(r'export KRB5_CONFIG.*\n?', '', content)
newbashfile = open(userenvironconfig, 'w')
newbashfile.write(content)
newbashfile.close()
except Exception:
pass
# remove symlinks for command line utilities
for utilname in self.binUtilities:
self.removeBinFile(utilname)
super(ZenPack, self).remove(app, leaveObjects=leaveObjects)
def get_ain_rules(dmd):
"""Get the AIN rules.
Only reloads from file once per minute. Once the rules are loaded they are
stored in a volatile (non-persistent) property of ``dmd``.
"""
last_ain_rules_load = getattr(dmd, '_v_last_ain_rules_load', 0)
if last_ain_rules_load > (time.time() - RELOAD_FREQUENCY):
return getattr(dmd, '_v_ain_rules', [])
else:
rules = []
rules_filename = zenPath('etc', 'ain_rules.py')
if os.path.isfile(rules_filename):
LOG.info("Reloading AIN rules from %s", rules_filename)
try:
rules_file = open(rules_filename, 'r')
for line in rules_file:
rule_wo_comment = line.split('#')[0].strip()
if rule_wo_comment != '':
rules.append(rule_wo_comment)
rules_file.close()
except Exception:
LOG.exception("Error loading AIN rules from %s", rules_filename)
else:
LOG.info("No AIN rules file found at %s", rules_filename)
# Store rules as a volatile property on dmd.
dmd._v_ain_rules = rules
dmd._v_last_ain_rules_load = time.time()
return rules
def _fetchRrdValues(self, rrdNames, rrd_paths, component, eventClass, severity):
rrdStart = 'now-600s'
rrdEnd = 'now'
perfDir = zenPath('perf')
log.debug("Datapoints to check: %s", rrdNames)
for rrdName in rrdNames:
try:
filePath = os.path.join(perfDir, rrd_paths[rrdName])
values = self._dataService.readRRD(filePath,
'AVERAGE',
"-s " + rrdStart,
"-e " + rrdEnd)[2]
except Exception, e:
log.debug("Unable to read RRD for dataPoint: %s", e)
continue
for value in reversed(values):
value = value[0]
if value is not None:
break
if value is None:
value = -1
message = "NAN for dataPoint %s on component %s on device %s!" % (rrdName, component, self._devId)
log.warn("%s", message)
evt = dict(device = self._devId, eventClass=eventClass, summary = message, component = component, severity = severity)
elif value is not None:
message = "NAN clear for dataPoint %s on component %s on device %s!" % (rrdName, component, self._devId)
evt = dict(device = self._devId, eventClass=eventClass, summary = message, component = component, severity = Event.Clear)
self._eventService.sendEvent(evt)
log.debug("Datapoint %s value: %s", rrdName, value)
def CreateZenPacksDir():
"""
Make sure $ZENHOME/ZenPacks exists
"""
zpDir = zenPath('ZenPacks')
if not os.path.isdir(zpDir):
os.mkdir(zpDir, 0750)
def _getDaemonPID(self, name):
"""
For a given daemon name, return its PID from a .pid file.
"""
if name == 'zenwebserver':
name = 'nginx'
elif name == 'zopectl':
name = 'Z2'
elif name == 'zeoctl':
name = 'ZEO'
elif '_' in name:
collector, daemon = name.split('_', 1)
name = '%s-%s' % (daemon, collector)
else:
name = "%s-localhost" % name
pidFile = zenPath('var', '%s.pid' % name)
if os.path.exists(pidFile):
pid = open(pidFile).read()
try:
pid = int(pid)
except ValueError:
return None
return self._pidRunning(int(pid))
else:
pid = None
return pid
def UploadZenPack(dmd, packName, project, description, znetUser, znetPass):
"""
Upload the specified zenpack to the given project.
Project is a string of the form 'enterprise/myproject' or
'community/otherproject'.
"""
zp = dmd.ZenPackManager.packs._getOb(packName, None)
if not zp:
raise ZenPackException('No ZenPack named %s' % packName)
# Export the zenpack
fileName = zp.manage_exportPack()
# Login to Zenoss.net
from DotNetCommunication import DotNetSession
session = DotNetSession()
session.login(znetUser, znetPass)
# Upload
zpFile = open(zenPath('export', fileName), 'r')
try:
response = session.open('%s/createRelease' % project.strip('/'), {
'description': description,
'fileStorage': zpFile,
})
finally:
zpFile.close()
if response:
result = response.read()
if "'success':true" not in result:
raise ZenPackException('Upload failed')
else:
raise ZenPackException('Failed to connect to Zenoss.net')
return
def config(self, repeat=False, fileName='deprecated.log', delay=True, propagate=True):
"""
Initialize or change the behavior of @deprecated.
@param repeat: Log every time the same deprecated function is called, or just once?
@type repeat: boolean
@param fileName: Name of the file, or None for no file.
@type fileName: string
@param delay: Prevent creating the log file until used?
@type delay: boolean
@param propagate: Also log to current file/screen?
@type propagate: boolean
"""
self.log = logging.getLogger('zen.deprecated')
# Remove the handler to start from scratch.
if self.log.handlers:
self.log.removeHandler(self.log.handlers[0])
# New settings
self.repeat = repeat
self.log.propagate = propagate
if fileName:
filePath = zenPath('log', fileName)
handler = logging.FileHandler(filePath, delay=delay)
# 2011-11-02 17:44:43,674 WARNING zen.deprecated: ...
formatter = logging.Formatter('%(asctime)s %(levelname)s %(name)s: %(message)s')
handler.setFormatter(formatter)
self.log.addHandler(handler)
else:
self.log.addHandler(logging.NullHandler())
return fp.read()
finally:
fp.close()
def cached(fname, cachedTime=600):
"return the contents of a file if it is young enough"
try:
if os.stat(fname).st_mtime > time.time() - cachedTime:
return read(fname)
except OSError:
return None
from Products.ZenUtils.Utils import zenPath
perf = zenPath('perf')
title = 'Plugin Title'
label = ''
width = 500
height = 100
start = '-7d'
end = 'now'
name = 'test'
def basicArgs(env):
args = [
'--imgformat=PNG',
'--start=%(start)s' % env,
'--end=%(end)s' % env,
from zope.component import getUtility
from AccessControl.SecurityManagement import newSecurityManager
from AccessControl.SecurityManagement import noSecurityManager
from Products.ZenUtils.Utils import getObjByPath, zenPath
from Products.ZenUtils.ZodbFactory import IZodbFactoryLookup
from Exceptions import ZentinelException
from ZenDaemon import ZenDaemon
from Products.ZenRelations.ZenPropertyManager import setDescriptors
from Products.ZenUtils.mysql import MySQLdb
from Products.ZenUtils.Utils import wait
defaultCacheDir = zenPath('var')
class DataRootError(Exception):
pass
def login(context, name='admin', userfolder=None):
"""Logs in."""
if userfolder is None:
userfolder = context.getPhysicalRoot().acl_users
user = userfolder.getUserById(name)
if user is None: return
if not hasattr(user, 'aq_base'):
user = user.__of__(userfolder)
newSecurityManager(None, user)
def run(self):
"""Execute the user's request"""
if self.args:
print "Require one of --install, --remove or --list flags."
self.parser.print_help()
return
if self.options.installPackName:
audit('Shell.ZenPack.Install',
zenpack=self.options.installPackName)
elif self.options.fetch:
audit('Shell.ZenPack.Fetch', zenpack=self.options.fetch)
elif self.options.removePackName:
audit('Shell.ZenPack.Remove', zenpack=self.options.removePackName)
if self.options.installPackName:
eggInstall = (self.options.installPackName.lower().endswith('.egg')
or os.path.exists(
os.path.join(self.options.installPackName,
'setup.py')))
# files-only just lays the files down and doesn't "install"
# them into zeo
class ZPProxy:
def __init__(self, zpId):
self.id = zpId
def path(self, *parts):
return zenPath('Products', self.id, *parts)
if self.options.installPackName and self.options.filesOnly:
if eggInstall:
return EggPackCmd.InstallZenPack(None,
self.options.installPackName,
filesOnly=True)
packName = self.extract(self.options.installPackName)
proxy = ZPProxy(packName)
for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
loader.load(proxy, None)
return
if self.options.removePackName and self.options.filesOnly:
# Remove files-only is not yet supported for egg zenpacks
# todo
proxy = ZPProxy(self.options.removePackName)
for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
loader.unload(proxy, None)
os.system('rm -rf %s' %
zenPath('Products', self.options.removePackName))
return
self.connect()
if not getattr(self.dmd, 'ZenPackManager', None):
raise ZenPackNeedMigrateException(
'Your Zenoss database appears'
' to be out of date. Try running zenmigrate to update.')
if (self.options.installPackName or
self.options.removePackName) and not self._verifyZepRunning():
print >> sys.stderr, "Error: Required daemon zeneventserver not running."
print >> sys.stderr, "Execute 'zeneventserver start' and retry the ZenPack installation."
sys.exit(1)
if self.options.installPackName:
if self.options.skipSameVersion and self._sameVersion():
return
if not self.preInstallCheck(eggInstall):
self.stop('%s not installed' % self.options.installPackName)
if eggInstall:
return EggPackCmd.InstallEggAndZenPack(
self.dmd,
self.options.installPackName,
link=self.options.link,
filesOnly=False,
previousVersion=self.options.previousVersion,
fromUI=self.options.fromui)
if os.path.isfile(self.options.installPackName):
packName = self.extract(self.options.installPackName)
elif os.path.isdir(self.options.installPackName):
if self.options.link:
packName = self.linkDir(self.options.installPackName)
else:
packName = self.copyDir(self.options.installPackName)
else:
self.stop(
'%s does not appear to be a valid file or directory.' %
self.options.installPackName)
# We want to make sure all zenpacks have a skins directory and that it
# is registered. The zip file may not contain a skins directory, so we
# create one here if need be. The directory should be registered
# by the zenpack's __init__.py and the skin should be registered
# by ZPLSkins loader.
skinsSubdir = zenPath('Products', packName, 'skins', packName)
if not os.path.exists(skinsSubdir):
os.makedirs(skinsSubdir, 0750)
self.install(packName)
elif self.options.fetch:
return EggPackCmd.FetchAndInstallZenPack(self.dmd,
self.options.fetch)
elif self.options.removePackName:
pack = self.dmd.ZenPackManager.packs._getOb(
self.options.removePackName, None)
if not pack:
if not self.options.ifinstalled:
self.log.info('ZenPack %s is not installed.' %
self.options.removePackName)
return False
else:
removeZenPackQueuesExchanges(pack.path())
if pack.isEggPack():
return EggPackCmd.RemoveZenPack(
self.dmd, self.options.removePackName)
RemoveZenPack(self.dmd, self.options.removePackName, self.log)
elif self.options.list:
for zpId in self.dmd.ZenPackManager.packs.objectIds():
try:
zp = self.dmd.ZenPackManager.packs._getOb(zpId, None)
except AttributeError:
zp = None
if not zp:
desc = 'broken'
elif zp.isEggPack():
desc = zp.eggPath()
else:
desc = zp.path()
print('%s (%s)' % (zpId, desc))
transaction.commit()
def fullname(partial):
return zenPath('perf', partial + '.rrd')
def run(self):
"""Execute the user's request"""
if self.args:
print "Require one of --install, --remove, --export, --create, or --list flags."
self.parser.print_help()
return
if self.options.installPackName:
audit('Shell.ZenPack.Install', zenpack=self.options.installPackName)
elif self.options.fetch:
audit('Shell.ZenPack.Fetch', zenpack=self.options.fetch)
elif self.options.exportPack:
audit('Shell.ZenPack.Export', zenpack=self.options.exportPack)
elif self.options.removePackName:
audit('Shell.ZenPack.Remove', zenpack=self.options.removePackName)
elif self.options.createPackName:
audit('Shell.ZenPack.Create', zenpack=self.options.createPackName)
ZenPack.currentServiceId = self.options.serviceId
if self.options.createPackName:
devDir, packName = os.path.split(self.options.createPackName)
try:
self.connect()
self.dmd.ZenPackManager.manage_addZenPack(packName, devDir=devDir)
except Exception as ex:
self.log.fatal("could not create zenpack: %s", ex)
sys.exit(1)
sys.exit(0)
if self.options.installPackName:
eggInstall = (self.options.installPackName.lower().endswith('.egg')
or os.path.exists(os.path.join(self.options.installPackName,
'setup.py')))
if self.options.removePackName and self.options.filesOnly:
# A files-only remove implies a files-only install. Egg packs are only supported here.
installedEggs = [ p for p in iter_entry_points('zenoss.zenpacks') ]
theOne = filter(lambda x: x.name == self.options.removePackName, installedEggs)
if not theOne:
raise ZenPackNotFoundException("Specified zenpack not installed")
if len(theOne) > 1:
raise ZenPackException("Multiple matching distributions for {} found - aborting.".format(self.options.removePackName))
actualZPDir = theOne[0].dist.location
class ZPProxy:
def __init__(self, zpId, actualPath):
self.id = zpId
self.actualPath = actualPath
def path(self, *parts):
return self.actualPath
proxy = ZPProxy(self.options.removePackName, actualZPDir)
for loader in (ZPL.ZPLDaemons(), ZPL.ZPLBin(), ZPL.ZPLLibExec()):
loader.unload(proxy, None)
try:
os.system('pip uninstall -y "%s"' % self.options.removePackName)
except Exception as ex:
self.log.error('Could not uninstall "%s"', self.options.removePackName)
raise ex # treat it as a fatal error
return
self.connect()
if not getattr(self.dmd, 'ZenPackManager', None):
raise ZenPackNeedMigrateException('Your Zenoss database appears'
' to be out of date. Try running zenmigrate to update.')
if (self.options.installPackName or self.options.removePackName) and not self._verifyZepRunning(timeout=600, delay=5):
print >> sys.stderr, "Error: Required daemon zeneventserver not running."
print >> sys.stderr, "Execute 'zeneventserver start' and retry the ZenPack installation."
sys.exit(1)
if self.options.installPackName:
# Process each install filter utility - order does not matter
for name, util in getUtilitiesFor(IZenPackInstallFilter):
# Get normalized pack name to compare. Split on path separator
# in case absolute path was given
egg_name = EGG_NAME(os.path.basename(os.path.normpath(self.options.installPackName)))
if not egg_name:
self.stop('Could not determine egg name from "%s"' % self.options.installPackName)
packName = egg_name.group('name')
if not util.installable(packName):
self.stop('Filter %s does not allow %s to be installed' \
% (name, self.options.installPackName))
if self.options.skipSameVersion and self._sameVersion():
return
if not self.preInstallCheck(eggInstall):
self.stop('%s not installed' % self.options.installPackName)
if eggInstall:
try:
return EggPackCmd.InstallEggAndZenPack(
self.dmd,
self.options.installPackName,
link=self.options.link,
filesOnly=self.options.filesOnly,
previousVersion= self.options.previousVersion,
fromUI=self.options.fromui,
serviceId=self.options.serviceId,
ignoreServiceInstall=self.options.ignoreServiceInstall)
except (OSError,) as e:
if self.options.link:
self.stop('%s cannot be installed with --link option' % self.options.installPackName)
else:
self.stop('%s could not be installed' % self.options.installPackName)
if os.path.isfile(self.options.installPackName):
packName = self.extract(self.options.installPackName)
elif os.path.isdir(self.options.installPackName):
if self.options.link:
packName = self.linkDir(self.options.installPackName)
else:
packName = self.copyDir(self.options.installPackName)
else:
self.stop('%s does not appear to be a valid file or directory.'
% self.options.installPackName)
# We want to make sure all zenpacks have a skins directory and that it
# is registered. The zip file may not contain a skins directory, so we
# create one here if need be. The directory should be registered
# by the zenpack's __init__.py and the skin should be registered
# by ZPLSkins loader.
skinsSubdir = zenPath('Products', packName, 'skins', packName)
if not os.path.exists(skinsSubdir):
os.makedirs(skinsSubdir, 0750)
self.install(packName)
elif self.options.fetch:
return EggPackCmd.FetchAndInstallZenPack(self.dmd, self.options.fetch)
elif self.options.exportPack:
return EggPackCmd.ExportZenPack(
self.dmd, self.options.exportPack)
elif self.options.removePackName:
try:
self.dmd.startPauseADM()
pack = self.dmd.ZenPackManager.packs._getOb(
self.options.removePackName, None)
if not pack:
if not self.options.ifinstalled:
self.log.info(
'ZenPack %s is not installed.',
self.options.removePackName
)
return False
else:
if pack:
removeZenPackQueuesExchanges(pack.path())
if pack.isEggPack():
return EggPackCmd.RemoveZenPack(
self.dmd, self.options.removePackName)
RemoveZenPack(self.dmd, self.options.removePackName, self.log)
finally:
self.dmd.stopPauseADM()
elif self.options.list:
for zpId in self.dmd.ZenPackManager.packs.objectIds():
try:
zp = self.dmd.ZenPackManager.packs._getOb(zpId, None)
except AttributeError:
zp = None
if not zp:
desc = 'broken'
elif zp.isEggPack():
try:
desc = zp.eggPath()
except DistributionNotFound:
desc = "zenpack missing"
else:
desc = zp.path()
print('%s (%s)' % (zpId, desc))
elif self.options.restore:
self.log.info("Restoring zenpacks")
self.restore()
transaction.commit()
##############################################################################
"""
Operations for Navigation
Available at: /zport/dmd/detailnav_router
"""
import logging
from Products.ZenUtils.Ext import DirectRouter
from Products.ZenUtils.extdirect.router import DirectResponse
from Products.Zuul.decorators import require
from Products.ZenUI3.security.security import permissionsForContext
from Products.ZenUtils.Utils import zenPath
log = logging.getLogger('zen.pagestats')
# create file handler which logs even debug messages
fh = logging.FileHandler(zenPath('log' + '/pagestats.log'))
# create formatter and add it to the handlers
formatter = logging.Formatter(
'%(asctime)s - %(name)s - %(levelname)s - %(message)s')
fh.setFormatter(formatter)
log.addHandler(fh)
class DetailNavRouter(DirectRouter):
"""
Router to Details navigation for given uid
"""
def _getLinkMenuItems(self, menuIds, ob):
def filterFn(menu):
return not menu.isdialog
# Copyright (C) Zenoss, Inc. 2012, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import os
import subprocess
from Products.ZenUtils.Utils import zenPath
logger = logging.getLogger('zen.callhome')
CRYPTPATH = zenPath('Products', 'ZenCallHome', 'transport', 'crypt')
GPGCMD = 'gpg --batch --no-tty --quiet --no-auto-check-trustdb '
def _getEnv():
env = os.environ.copy()
env.pop('GPG_AGENT_INFO', None)
return env
def encrypt(stringToEncrypt, publicKey):
cmd = (GPGCMD + '--keyring %s --trustdb-name %s -e -r %s' %
(CRYPTPATH + '/pubring.gpg', CRYPTPATH + '/trustdb.gpg', publicKey))
p = subprocess.Popen(cmd,
shell=True,
def connect(self):
zopeconf = zenPath("etc","zope.conf")
import Zope2
Zope2.configure(zopeconf)
self.app = Zope2.app()
def build(self):
self.db = None
self.storage = None
conn = None
try:
self.zodbConnect()
conn = self.db.open()
root = conn.root()
app = root.get('Application')
if app and getattr(app, self.sitename, None) is not None:
print "zport portal object exists; exiting."
return
except self.connectionFactory.exceptions.OperationalError as e:
print "zenbuild: Database does not exist."
sys.exit(1)
finally:
if conn:
conn.close()
if self.db:
self.db.close()
self.db = None
if self.storage:
self.storage.close()
self.storage = None
# TODO: remove the port condition, in here for now because there is no SQL dump of postgresql
if self.options.fromXml:
self.connect()
from Products.ZenModel.ZentinelPortal import manage_addZentinelPortal
manage_addZentinelPortal(self.app, self.sitename)
site = self.app._getOb(self.sitename)
# build index_html
if self.app.hasObject('index_html'):
self.app._delObject('index_html')
from Products.PythonScripts.PythonScript import manage_addPythonScript
manage_addPythonScript(self.app, 'index_html')
newIndexHtml = self.app._getOb('index_html')
text = 'container.REQUEST.RESPONSE.redirect(container.zport.virtualRoot() + "/zport/dmd/")\n'
newIndexHtml.ZPythonScript_edit('', text)
# build standard_error_message
if self.app.hasObject('standard_error_message'):
self.app._delObject('standard_error_message')
file = open(zenPath('Products/ZenModel/dtml/standard_error_message.dtml'))
try:
text = file.read()
finally:
file.close()
import OFS.DTMLMethod
OFS.DTMLMethod.addDTMLMethod(self.app, id='standard_error_message',
file=text)
# Convert the acl_users folder at the root to a PAS folder and update
# the login form to use the Zenoss login form
Security.replaceACLWithPAS(self.app, deleteBackup=True)
auth0_setup(self.app)
account_locker_setup(self.app)
# Add groupManager to zport.acl
acl = site.acl_users
if not hasattr(acl, 'groupManager'):
plugins.ZODBGroupManager.addZODBGroupManager(acl, 'groupManager')
acl.groupManager.manage_activateInterfaces(['IGroupsPlugin',])
trans = transaction.get()
trans.note("Initial ZentinelPortal load by zenbuild.py")
trans.commit()
print "ZentinelPortal loaded at %s" % self.sitename
# build dmd
from Products.ZenModel.DmdBuilder import DmdBuilder
dmdBuilder = DmdBuilder(site,
self.options.evthost,
self.options.evtuser,
self.options.evtpass,
self.options.evtdb,
self.options.evtport,
self.options.smtphost,
self.options.smtpport,
self.options.pagecommand)
dmdBuilder.build()
transaction.commit()
# Load XML Data
from Products.ZenModel.XmlDataLoader import XmlDataLoader
dl = XmlDataLoader(noopts=True, app=self.app)
dl.loadDatabase()
else:
cmd = "gunzip -c %s | %s --usedb=zodb" % (
zenPath("Products/ZenModel/data/zodb.sql.gz"),
zenPath("Products/ZenUtils/ZenDB.py"),
)
returncode = os.system(cmd)
if returncode:
print >> sys.stderr, "There was a problem creating the database from the sql dump."
sys.exit(1)
# Relstorage may have already loaded items into the cache in the
# initial connection to the database. We have to expire everything
# in the cache in order to prevent errors with overlapping
# transactions from the model which was just imported above.
if self.options.zodb_cacheservers:
self.flush_memcached(self.options.zodb_cacheservers.split())
self.connect()
# Set all the attributes
site = getattr(self.app, self.sitename, None)
site.dmd.smtpHost = self.options.smtphost
site.dmd.smtpPort = self.options.smtpport
site.dmd.pageCommand = self.options.pagecommand
site.dmd.uuid = None
for evmgr in (site.dmd.ZenEventManager, site.dmd.ZenEventHistory):
evmgr.username = self.options.evtuser
evmgr.password = self.options.evtpass
evmgr.database = self.options.evtdb
evmgr.host = self.options.evthost
evmgr.port = self.options.evtport
transaction.commit()
# Load reports
from Products.ZenReports.ReportLoader import ReportLoader
rl = ReportLoader(noopts=True, app=self.app)
rl.loadDatabase()
def _getConfigFilename(self, daemon):
if daemon in ('zopectl', 'zenwebserver'):
daemon = 'zope'
elif daemon == 'zeoctl':
daemon = 'zeo'
return zenPath('etc', "%s.conf" % daemon)
def __init__(self):
"""Initialize a ConnectedToZenHubSignalFile instance."""
filename = "zenhub_connected"
self.__signalFilePath = zenPath('var', filename)
self.__log = getLogger(self)
def _getSourcePath(self):
return zenPath(self.sourcepath)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
changed = False
# If the service lacks Solr, add it now.
solr = filter(lambda s: s.name == "Solr", ctx.services)
log.info("Found %i services named 'Solr'." % len(solr))
if not solr:
imageid = os.environ['SERVICED_SERVICE_IMAGE']
log.info("No Solr found; creating new service.")
new_solr = default_solr_service(imageid)
infrastructure = ctx.findServices('^[^/]+/Infrastructure$')[0]
ctx.deployService(json.dumps(new_solr), infrastructure)
changed = True
# Remove the zencatalogservice-uri global conf option from the top service
zenoss = ctx.getTopService()
global_conf = zenoss.context
if global_conf and "global.conf.zencatalogservice-uri" in global_conf:
del global_conf["global.conf.zencatalogservice-uri"]
changed = True
# Now healthchecks
solr_answering_healthcheck = HealthCheck(
name="solr_answering",
interval=10.0,
script=
"curl -A 'Solr answering healthcheck' -s http://localhost:8983/solr/zenoss_model/admin/ping?wt=json | grep -q '\"status\":\"OK\"'"
)
for svc in ctx.services:
# Remove zencatalogservice, if it still exists
if svc.name == "zencatalogservice":
svcid = svc._Service__data['ID']
# ZEN-28127: during unittests, ctx is an instance of
# FakeServiceContext which is set to None.
if ctx._client is not None:
ctx._client.deleteService(svcid)
ctx.services.remove(svc)
changed = True
continue
# Remove zencatalogservice response prereq and add solr one
for pr in svc.prereqs[:]:
if pr.name == "zencatalogservice response":
svc.prereqs.remove(pr)
svc.prereqs.append(
sm.Prereq(
name='Solr answering',
script=
"""curl -A 'Solr answering prereq' -s http://localhost:8983/solr/zenoss_model/admin/ping?wt=json | grep -q '\"status\":\"OK\"'"""
))
changed = True
# If we've got a solr_answering health check, we can stop.
# Otherwise, remove catalogservice health checks and add Solr ones
if filter(lambda c: c.name == 'solr_answering', svc.healthChecks):
continue
for hc in svc.healthChecks:
if hc.name == "catalogservice_answering":
svc.healthChecks.remove(hc)
changed = True
# Get rid of erroneous "solr" endpoints that some older migrations added and the "zodb_zencatalogservice" endpoint import that zenimpactstate may have
eps_to_remove = filter(
lambda ep: ep.purpose == 'import' and
(ep.application == 'solr' or ep.application ==
'zodb_zencatalogservice'), svc.endpoints)
for ep in eps_to_remove:
changed = True
svc.endpoints.remove(ep)
for ep in svc.endpoints:
if ep.purpose == 'import' and ep.application == 'zodb_.*':
svc.healthChecks.append(solr_answering_healthcheck)
changed = True
break
filterName = 'solr'
filename = 'Products/ZenModel/migrate/data/%s-6.0.0.conf' % filterName
with open(zenPath(filename)) as filterFile:
try:
filterDef = filterFile.read()
except Exception, e:
log.error("Error reading {0} logfilter file: {1}".format(
filename, e))
return
log.info("Updating log filter named {0}".format(filterName))
changed = True
ctx.addLogFilter(filterName, filterDef)
##############################################################################
#
# Copyright (C) Zenoss, Inc. 2007, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import socket
import time
import Globals
from Products.ZenUtils.Utils import zenPath
defaultInfile = zenPath("log/origsyslog.log")
from Products.ZenUtils.CmdBase import CmdBase
SYSLOG_PORT = socket.getservbyname('syslog', 'udp')
class ZenSendSyslog(CmdBase):
def __init__(self):
CmdBase.__init__(self)
self.host = socket.gethostbyname(self.options.host)
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
def run(self):
self.log.info("sending messages to host %s", self.host)
count = 0
start = time.time()
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
top = ctx.getTopService()
log.info("Found top level service: '{0}'".format(top.name))
if top.name.find("Zenoss") != 0 and top.name.find("UCS-PM") != 0:
log.info(
"Top level service name isn't Zenoss or UCS-PM; skipping.")
return
services = filter(lambda s: "opentsdb" in ctx.getServicePath(s),
ctx.services)
log.info("Found %i services with 'opentsdb' in their service path." %
len(services))
changed = False
filterName = "hbasedaemon"
for service in services:
servicePath = ctx.getServicePath(service)
# First, fix the path and the log filter name
for logConfig in service.logConfigs:
if logConfig.path == "/opt/zenoss/log/opentsdb.log":
log.info("Updating logfilter for %s in %s", logConfig.path,
servicePath)
logConfig.path = "/var/log/opentsdb/opentsdb.log"
logConfig.filters = [filterName]
changed = True
else:
log.info("No updates necesary for the logfilter for %s",
servicePath)
# Second, update the log file config to generate messages in the same format as hbase
configFiles = service.originalConfigs + service.configFiles
hbaseLogPattern = "%date{ISO8601} %-5level [%thread] %logger{0}: %msg%n"
for configFile in filter(
lambda f: f.name == '/opt/opentsdb/src/logback.xml',
configFiles):
if "%d{HH:mm:ss.SSS} %-5level [%logger{0}.%M] - %msg%n" in configFile.content:
log.info("Updating /opt/opentsdb/src/logback.xml for %s",
servicePath)
configFile.content = configFile.content.replace(
"%d{ISO8601} %-5level [%thread] %logger{0}: %msg%n",
hbaseLogPattern)
configFile.content = configFile.content.replace(
"%d{HH:mm:ss.SSS} %-5level [%logger{0}.%M] - %msg%n",
hbaseLogPattern)
configFile.content = configFile.content.replace(
"%date{ISO8601} [%logger.%M] %msg%n", hbaseLogPattern)
changed = True
filename = 'Products/ZenModel/migrate/data/hbasedaemon-6.0.0.conf'
with open(zenPath(filename)) as filterFile:
try:
filterDef = filterFile.read()
except Exception, e:
log.error("Error reading {0} logfilter file: {1}".format(
filename, e))
return
log.info("Updating log filter named {0}".format(filterName))
changed = True
ctx.addLogFilter(filterName, filterDef)
def buildOptions(self):
self.parser.add_option('--create',
dest='createPackName',
default=None,
help="Zenpack name or path to full destination path, eg /home/zenoss/src/ZenPacks.example.TestPack")
self.parser.add_option('--install',
dest='installPackName',
default=None,
help="Path to the ZenPack to install.")
self.parser.add_option('--fetch',
dest='fetch',
default=None,
help="Name of ZenPack to retrieve from Zenoss and install")
self.parser.add_option('--export',
dest='exportPack',
default=None,
help="Name of the ZenPack to export.")
self.parser.add_option('--remove', '--delete', '--uninstall', '--erase',
dest='removePackName',
default=None,
help="Name of the ZenPack to remove.")
self.parser.add_option('--restore',
dest='restore',
action="store_true",
default=False,
help='restore missing zenpacks')
self.parser.add_option('--list',
dest='list',
action="store_true",
default=False,
help='List installed ZenPacks')
self.parser.add_option('--keep-pack',
dest='keepPackNames',
action='append',
type='string',
default=[],
help='(Can be used multiple times) with --restore, pack name to not be deleted from disk if not found in zodb')
self.parser.add_option('--link',
dest='link',
action="store_true",
default=False,
help="Install the ZenPack in place, without "
"copying into $ZENHOME/ZenPacks.")
self.parser.add_option('--files-only',
dest='filesOnly',
action="store_true",
default=False,
help='Install the ZenPack files onto the '
'filesystem, but do not install the '
'ZenPack into Zenoss.')
self.parser.add_option('--fromui',
dest='fromui',
action="store_true",
default=False,
help=optparse.SUPPRESS_HELP)
self.parser.add_option('--previousversion',
dest='previousVersion',
default=None,
help="Previous version of the zenpack;"
' used during upgrades')
self.parser.add_option('--if-installed',
action="store_true",
dest='ifinstalled',
default=False,
help="Delete ZenPack only if installed")
self.parser.add_option('--skip-same-version',
action="store_true",
dest='skipSameVersion',
default=False,
help="Do not install the zenpack if the version is unchanged")
self.parser.add_option('--service-id',
dest='serviceId',
default=os.getenv('CONTROLPLANE_SERVICED_ID', ''),
help=optparse.SUPPRESS_HELP)
self.parser.add_option('--ignore-service-install',
dest='ignoreServiceInstall',
action='store_true',
default=False,
help='On install of a zenpack, do not attempt to install '
'any associated services into the control center')
self.parser.prog = "zenpack"
ZenScriptBase.buildOptions(self)
self.parser.defaults['logpath'] = zenPath('log')
def binPath(self, bin_file):
args = ['bin']
if '/zenrun.d/' in bin_file:
args.append('zenrun.d')
args.append(os.path.basename(bin_file))
return zenPath(*args)
from Products.ZenCollector.tasks import BaseTask, TaskStates
from Products.ZenCollector import interfaces
from Products.ZenCollector.tasks import SimpleTaskFactory
from Products.ZenUtils.Utils import zenPath
from Products.ZenEvents import ZenEventClasses
from zenoss.protocols.protobufs import zep_pb2 as events
# imports from within ZenStatus
from Products.ZenStatus import PingTask
from Products.ZenStatus.ping.CmdPingTask import CmdPingTask
from PingResult import PingResult, parseNmapXmlToDict
from Products.ZenStatus.PingCollectionPreferences import PingCollectionPreferences
from Products.ZenStatus.interfaces import IPingTaskFactory, IPingTaskCorrelator
from Products.ZenStatus import nmap
_NMAP_BINARY = zenPath("bin/nmap")
_CLEAR = 0
_CRITICAL = 5
_WARNING = 3
MAX_PARALLELISM = 150
DEFAULT_PARALLELISM = 10
# amount of IPs/events to process before giving time to the reactor
_SENDEVENT_YIELD_INTERVAL = 100 # should always be >= 1
# amount of time since last ping down before count is removed from dictionary
DOWN_COUNT_TIMEOUT_MINUTES = 15
# twisted.callLater has trouble with sys.maxint as call interval,
skinsDir = os.path.join(os.path.dirname(__file__), 'skins')
from Products.CMFCore.DirectoryView import registerDirectory
if os.path.isdir(skinsDir):
registerDirectory(skinsDir, globals())
from Products.ZenWidgets import messaging
from Products.ZenUtils.Utils \
import monkeypatch, zenPath, binPath, executeCommand, executeStreamCommand
from Products.ZenModel.PerformanceConf import performancePath
log = logging.getLogger('.'.join(['zen', __name__]))
COMMAND_TIMEOUT = 240
MASTER_DAEMON_LIST_FILE = zenPath('etc/master_daemons.txt')
DEFAULT_MASTER_DAEMONS = [
'zeoctl', 'zopectl', 'zeneventd', 'zenhub', 'zenjobs', 'zenactions',
'zenactiond'
]
zpDir = zenPath('ZenPacks')
updConfZenBin = zenPath('bin/updateConfigs')
updConfBin = os.path.join(os.path.dirname(__file__), 'bin/updateConfigs')
class UpdateRemoteCollectorView(StreamingView):
def write(self, data=''):
StreamingView.write(
self, ' '.join((strftime('%Y-%m-%d %H:%M:%S,000'), 'INFO',
def varPath(*args):
all_args = ['var'] + list(args)
return zenPath(*all_args)
class rancidd(CyclingDaemon):
# RANCID VERSION (as downloaded from shrubbery.net)
RANCID_VERSION = "2.3.8"
startDelay = 0 # in seconds
standalone = True
base_path = zenPath()
rancid_path = zenPath('rancid')
homedir_path = os.path.expanduser('~zenoss')
cloginFile = ".cloginrc"
cCustomLoginFile = "etc/cloginrc-custom"
configFile = "rancid.conf"
customConfigFile = "rancid-custom.conf"
cronFile = "zenoss-rancid.cron"
_list_of_groups = [] # list of rancid groups
_rancid = [] # list of all rancid-enabled devices
def __init__(self, standalone=True):
if standalone:
self.initStandalone()
else:
self.standalone = False
def initStandalone(self):
CyclingDaemon.__init__(self)
log.info("call init")
self.name = "zenrancidtest"
# in daemon mode we only want to run the process once a day by default
if self.options.daemon:
self._calcStartDelay()
log.debug("options: %s", self.options)
def getRancidVersion(self):
return self.RANCID_VERSION
## override run() to introduce the delay timer
def run(self):
reactor.callLater(self.startDelay, self.runCycle)
reactor.run()
def main_loop(self):
#print "MAIN LOOP STARTED"
log.info("main_loop started in rancidd")
# find out which devices to use
self.getRancidDevices()
# always re-create the rancid config file
self.generate_conf()
# always re-recreate the default .cloginrc file
self.generate_cloginrc()
# always run rancid-cvs to update CVS dirs before running rancid-run
# needed in case new groups have been added to the rancid config file
self.run_rancid_cvs()
# generate the router.db file
self.generate_router_db()
# rancid-run to get all configs
self.run_rancid()
def setup(self):
# find out which devices to use
#_devices, _groups = self.getRancidDevices()
#self.generate_crontab()
self.generate_cloginrc()
self.generate_conf()
self.generateCustomDbPath()
#self.generate_rancid_svn_binary()
self.generate_rancid_logrotate_file()
def buildOptions(self):
CyclingDaemon.buildOptions(self)
self.parser.add_option( '--device', dest='rDevice', type='string', default='', help='Run Rancid for a single device')
self.parser.add_option( '--rcssys', dest='rancid_rcssys', type='string', default="svn", help='Rancid repository, use either SVN or CVS - default SVN (not yet supported)')
self.parser.add_option( '--starttime', dest='starttime', type='string', default="02:00", help='Rancid start time. Format = HH:MM (default 02:00)')
self.parser.add_option( '--repeat', dest='repeat', type='int', default=86400, help='Rancid cycle time in seconds, run only once a day by default. (default 86400)')
# by default we want to run Rancid only once a day at 02:00
# cycle and starttime can be configured in the daemon conf file or set using commandline parameters
def _calcStartDelay(self):
currTime = strftime("%H:%M", localtime())
#startTime = '23:58'
startTime = self.options.starttime or '02:00'
self.options.cycletime = self.options.repeat or 60*60*4
FMT = '%H:%M'
delta = datetime.strptime(startTime, FMT) - datetime.strptime(currTime, FMT)
self.startDelay = delta.seconds
log.info("Rancid will start at %s (%s seconds to go). Cycle time = %s", startTime, self.startDelay, self.options.cycletime)
def getRancidDevices(self):
# initialize variables
self._list_of_groups = []
self._rancid = []
devs = self.dmd.Devices
devlist = devs.getSubDevices()
for dev in devlist:
log.debug("Found device: %s", dev)
## only use monitored devices
if not dev.monitorDevice(): continue
## only continue if Rancid is configured and enabled
if dev.getProperty("zRancidIgnore", False): continue
d = { 'id': dev.id,
'user': dev.getProperty("zRancidUser"),
'password': self.generate_password(dev.getProperty("zRancidPassword")),
'enablePassword': self.generate_password(dev.getProperty("zRancidEnablePassword")),
'group': dev.getProperty("zRancidGroup"),
'deviceType': dev.getProperty("zRancidDeviceType"),
'useSSH': dev.getProperty("zRancidSSH")
}
self._rancid.append(d)
if dev.getProperty("zRancidGroup") is not None:
self._list_of_groups.append(dev.getProperty("zRancidGroup"))
# make list unique
self._list_of_groups = list(set(self._list_of_groups))
log.debug("_rancid list = %s", self._rancid)
log.debug("_list_of_groups list = %s", self._list_of_groups)
def generate_password(self, password):
pwd = re.sub('\$', '\\$', password)
return pwd
def generate_cloginrc(self):
clogin = os.path.join(self.homedir_path, self.cloginFile)
out = open(clogin, "w")
## generate device specific info
for dev in self._rancid:
if not dev['id']: continue
dev_lc = dev['id'].lower()
out.write("## " + dev_lc + " ##\n")
if dev['user']:
out.write("add user " + dev_lc + " " + dev['user'] + "\n")
if dev['password']:
out.write("add password " + dev_lc + " " + dev['password'])
if dev['enablePassword']:
out.write(" " + dev['enablePassword'])
out.write("\n")
if dev['useSSH']:
out.write("add method %s ssh telnet\n" % (dev_lc))
out.write("\n")
## generate global defaults
out.write("include " + os.path.join(self.rancid_path, self.cCustomLoginFile) + "\n")
out.write("## GLOBAL DEFAULT ##\n")
out.write("add method * telnet ssh\n")
out.close
os.chmod(clogin, S_IREAD | S_IWRITE)
## generate a cloginrc that can be used for customized settings
self.generate_customized_cloginrc()
# create the framework for the custom DB files
def generateCustomDbPath(self):
path = zenPath('rancid', 'etc','customdb')
if not os.path.isdir(path):
os.mkdir(path, 0750)
out = open(zenPath('rancid', 'etc','customdb', 'readme'), 'w')
out.write("# create your own custom router.db files for devices which are not managed in Zenoss\n")
out.write("# create the same folder structure as exists in rancid/var and put your own router.db file inside\n")
out.close()
# TODO: add custom router.db
def generate_router_db(self):
db_files = {}
customdb_files = {}
for dev in self._rancid:
if not (dev['id'] and dev['group'] and dev['deviceType']): continue
groupDir = zenPath('rancid', 'var', dev['group'])
routerDbFile = os.path.join(groupDir, "router.db")
if os.path.exists(groupDir):
if not routerDbFile in db_files:
db_files[routerDbFile] = []
# store router.db in lower case
db_files[routerDbFile].append(":".join((dev['id'], dev['deviceType'], "up")))
for file in db_files:
# hack to force lower case
out = open(file, 'w')
for router in db_files[file]:
out.write(str(router).lower() + "\n")
# check if a custom router.db file exists in rancid/etc/customdb
customDbFile = re.sub(zenPath('rancid', 'var'), zenPath('rancid', 'etc', 'customdb'), file)
if os.path.exists(customDbFile):
with open(customDbFile) as f:
for line in f.readlines():
out.write(str(line))
out.close()
def generate_customized_cloginrc(self):
custom_clogin = os.path.join(self.rancid_path, self.cCustomLoginFile)
if not os.path.exists(custom_clogin):
out = open(custom_clogin, 'w')
out.write("## save customized changes to .cloginrc below this line ##\n")
out.close()
os.chmod(custom_clogin, S_IREAD | S_IWRITE)
def generate_rancid_logrotate_file(self):
file = os.path.join(self.rancid_path, 'etc', 'zenoss-rancid.logrotate')
logfiles = os.path.join(self.rancid_path, 'var/logs/')
rotateFile = "%s {\n" \
"missingok\n" \
"weekly\n" \
"rotate 2\n" \
"copytruncate\n" \
"}" % (logfiles)
if os.path.exists(os.path.join(self.rancid_path, 'etc')):
out = open(file, 'w')
out.write(rotateFile)
out.close()
# TODO: create rancid events
def run_rancid(self, dev='', grp=''):
if self.options.rDevice:
# try to find the device in zenoss
device = self.dmd.Devices.findDevice(self.options.rDevice)
if not device:
log.error('FAILED: device not found in Zenoss: %s', self.options.rDevice)
return False
group = device.zRancidGroup
id = device.id.lower()
type = device.zRancidDeviceType
log.info('Starting Rancid for device %s', id)
cmd = os.path.join(self.rancid_path, 'bin/rancid-run') + ' -r ' + id + ' ' + group
else:
cmd = os.path.join(self.rancid_path, 'bin/rancid-run')
#os.system("$ZENHOME/bin/zensendevent --severity Info -p rancid -c /App/Rancid rancid-cvs has finished")
log.debug('running command: "%s"', cmd)
if os.system(cmd):
os.system("$ZENHOME/bin/zensendevent --severity Error -p rancid -c /App/Rancid rancid-run has failed")
return - 1
else:
os.system("$ZENHOME/bin/zensendevent --severity Info -p rancid -c /App/Rancid rancid-run has finished successfully")
def run_rancid_cvs(self):
cmd = os.path.join(self.rancid_path, 'bin/rancid-cvs')
if os.system(cmd):
os.system("$ZENHOME/bin/zensendevent --severity Error -p rancid -c /App/Rancid rancid-cvs has failed")
return - 1
else:
os.system("$ZENHOME/bin/zensendevent --severity Info -p rancid -c /App/Rancid rancid-cvs has finished successfully")
def generate_conf(self):
rcssys = "svn"
if self.standalone:
rcssys = self.options.rancid_rcssys
## TODO: add version parameter here
template = "# rancid %s\n" \
"LD_LIBRARY_PATH=;\n" \
"TERM=network;export TERM\n" \
"umask 027\n" \
"TMPDIR=/tmp; export TMPDIR\n" \
"RANCIDDIR=%s; export RANCIDDIR\n" \
"BASEDIR=$RANCIDDIR/var; export BASEDIR\n" \
"PATH=$PATH:$RANCIDDIR/bin:/usr/bin:/usr/sbin:.:/bin:/usr/local/bin:/usr/bin; export PATH\n" \
"CVSROOT=$BASEDIR/CVS; export CVSROOT\n" \
"LOGDIR=$BASEDIR/logs; export LOGDIR\n" \
"RCSSYS=%s; export RCSSYS\n" \
"#NOPIPE=YES; export NOPIPE\n" \
"#FILTER_PWDS=YES; export FILTER_PWDS\n" \
"#NOCOMMSTR=YES; export NOCOMMSTR\n" \
"#MAX_ROUNDS=4; export MAX_ROUNDS\n" \
"#OLDTIME=4; export OLDTIME\n" \
"#LOCKTIME=4; export LOCKTIME\n" \
"#PAR_COUNT=5; export PAR_COUNT\n" \
"LIST_OF_GROUPS=\"%s\"\n" \
"# rancid-group: joe,moe@foo\n" \
"# rancid-admin-group: hostmaster\n" \
"#MAILDOMAIN=\"@example.com\"; export MAILDOMAIN\n" \
"#MAILHEADERS=\"Precedence: bulk\"; export MAILHEADERS\n" \
"# this file will be overwritten, if you want to customize it, override changes in the following file\n" \
". $RANCIDDIR/etc/%s\n" % (self.RANCID_VERSION, self.rancid_path, rcssys, " ".join(set(self._list_of_groups)), self.customConfigFile )
out = open(os.path.join(self.rancid_path, 'etc', self.configFile), 'w')
out.write(template)
out.close()
## make sure the imported customized conf file exists
self.generate_customized_conf()
def generate_customized_conf(self):
confFile = os.path.join(self.rancid_path, 'etc', self.customConfigFile)
if not os.path.exists(confFile):
out = open(confFile, 'w')
out.write("## save customized changes to rancid.conf below this line ##\n")
out.close()
ZenPack, ZenPackException,
ZenPackNotFoundException,
ZenPackNeedMigrateException
)
from Products.ZenUtils.ZenScriptBase import ZenScriptBase
from Products.ZenUtils.Utils import cleanupSkins, zenPath, binPath, varPath, get_temp_dir
import Products.ZenModel.ZenPackLoader as ZPL
from Products.ZenModel.ZenPackLoader import CONFIG_FILE, CONFIG_SECTION_ABOUT
from Products.ZenModel.ZVersion import VERSION
import ZenPackCmd as EggPackCmd
from Products.Zuul import getFacade
from zope.component import getUtilitiesFor
from Products.ZenUtils.ZenPackInstallFilter import IZenPackInstallFilter
ZPHISTORY = zenPath('zphistory.json')
UPGRADE_FROM_FILE = varPath('upgrade_from_version.txt')
HIGHER_THAN_CRITICAL = 100
LSB_EXITCODE_PROGRAM_IS_NOT_RUNNING = 3
def RemoveZenPack(dmd, packName, log=None,
skipDepsCheck=False, leaveObjects=True,
deleteFiles=True):
if log:
log.debug('Removing Pack "%s"', packName)
if not skipDepsCheck:
for pack in dmd.ZenPackManager.packs():
if packName in pack.requires:
raise ZenPackException('Pack %s depends on pack %s, '
def _getCountersFile(self):
return zenPath('var/%s_%s.pickle' % (
self.name,
self.options.monitor,
))
def binPath(self, daemon):
return zenPath('bin', os.path.basename(daemon))
def install(self, silent=False):
databases = self.options.databases
if not databases: databases = self._zodb_databases()
script = zenPath("Products", "ZenUtils", "relstorage", "mysql",
"003.sql")
with open(script, 'r') as f:
sql = f.read()
for database in databases:
check_sql = """
SELECT COUNT(*)
FROM information_schema.columns
WHERE table_schema = DATABASE()
AND UPPER(table_name) = 'CONNECTION_INFO';
"""
stdout, stderr = self._zendb(database, check_sql)
try:
if int(stdout) > 0:
if not silent:
LOG.info(
"Database already has %s.connection_info table. No action required."
% (database, ))
else:
stdout, stderr = self._zendb(database, sql)
if not silent:
LOG.warn(
"Created %s.connection_info table in database. PLEASE RESTART ALL DEPENDENT SERVICES."
% (database, ))
except ValueError:
if not silent:
LOG.error(
"Unable to determine if %s.connection_info table exists in database!"
% (database, ))
exit(1)
check_sql = """
SELECT COUNT(*)
FROM information_schema.routines
WHERE UPPER(routine_schema) = 'MYSQL'
AND UPPER(routine_name) = 'KILLTRANSACTIONS';
"""
sql = """
DROP PROCEDURE IF EXISTS mysql.KillTransactions;
DELIMITER $$
CREATE PROCEDURE mysql.KillTransactions(IN num_seconds int(10) unsigned, IN action varchar(10))
BEGIN
DECLARE _ts int;
DECLARE _q_db longtext;
DECLARE _q_pid longtext;
DECLARE _q_info longtext;
DECLARE _q_outer_joins longtext;
DECLARE done BOOL DEFAULT FALSE;
DECLARE v_db varchar(64);
DECLARE _txns_count INT DEFAULT 0;
DECLARE _txns_killed TEXT DEFAULT "'HOST','DB','COMMAND','STATE','INFO','PID','LINE1','LINE2','TRX_ID','TRX_QUERY','TRX_STARTED','TRX_MYSQL_THREAD_ID'";
DECLARE v_host varchar(54);
DECLARE v_command varchar(16);
DECLARE v_state varchar(64);
DECLARE v_info longtext;
DECLARE v_pid bigint(21) unsigned;
DECLARE v_line1 varchar(1000);
DECLARE v_line2 varchar(1000);
DECLARE v_trx_id varchar(20);
DECLARE v_trx_query varchar(1024);
DECLARE v_started varchar(20);
DECLARE v_thread_id bigint(21) unsigned;
DECLARE c_db CURSOR FOR
SELECT DISTINCT p.db
FROM information_schema.innodb_trx trx
INNER JOIN information_schema.processlist p
ON p.id = trx.trx_mysql_thread_id
WHERE (_ts - unix_timestamp(trx.trx_started)) > num_seconds;
DECLARE c_thread_id CURSOR FOR
SELECT *
FROM long_transactions
WHERE (_ts - unix_timestamp(trx_started)) > num_seconds;
DECLARE CONTINUE HANDLER FOR SQLSTATE '02000' SET done = TRUE;
SET done = FALSE;
SET _ts = unix_timestamp();
OPEN c_db;
REPEAT
FETCH c_db INTO v_db;
IF NOT done AND EXISTS(SELECT 1
FROM information_schema.columns
WHERE table_schema = v_db
AND UPPER(table_name) = 'CONNECTION_INFO')
THEN
SET _q_db = CONCAT('`',REPLACE(v_db,'`','``'),'`');
SET _q_pid = CONCAT_WS(', ', _q_pid,
CONCAT(_q_db, '.connection_info.pid'));
SET _q_info = CONCAT_WS(', ', _q_info,
CONCAT(_q_db, '.connection_info.info'));
SET _q_outer_joins = CONCAT_WS(' ', _q_outer_joins,
CONCAT('LEFT OUTER JOIN ',
_q_db,
'.connection_info on p.id = ',
_q_db,
'.connection_info.connection_id and p.db = ',
QUOTE(v_db)));
END IF;
UNTIL done END REPEAT;
SET @query = CONCAT('
CREATE OR REPLACE VIEW
long_transactions
AS
SELECT
p.host,
p.db,
p.command,
p.state,
p.info,
coalesce(', coalesce(_q_pid,'NULL'), ') as pid,
substring_index(coalesce(', coalesce(_q_info,'NULL'), '),''\n'',1) as line1,
substring_index(substring_index(coalesce(', coalesce(_q_info,'NULL'), '),''\n'',2),''\n'',-1) as line2,
trx.trx_id,
trx.trx_query,
trx.trx_started,
trx.trx_mysql_thread_id
FROM information_schema.innodb_trx trx
INNER JOIN information_schema.processlist p
ON p.id = trx.trx_mysql_thread_id ',
coalesce(_q_outer_joins,''));
PREPARE stmt FROM @query;
EXECUTE stmt;
DEALLOCATE PREPARE stmt;
SET done = FALSE;
OPEN c_thread_id;
REPEAT
FETCH c_thread_id
INTO
v_host,
v_db,
v_command,
v_state,
v_info,
v_pid,
v_line1,
v_line2,
v_trx_id,
v_trx_query,
v_started,
v_thread_id;
IF NOT done THEN
SET _txns_killed = CONCAT_WS('\n', _txns_killed, CONCAT_WS(',',
QUOTE(v_host),
QUOTE(coalesce(v_db,'')),
QUOTE(v_command),
QUOTE(coalesce(v_state,'')),
QUOTE(coalesce(v_info,'')),
QUOTE(coalesce(v_pid,'')),
QUOTE(coalesce(v_line1,'')),
QUOTE(coalesce(v_line2,'')),
QUOTE(v_trx_id),
QUOTE(coalesce(v_trx_query,'')),
QUOTE(v_started),
QUOTE(v_thread_id)));
IF 'KILL' = upper(action) THEN
KILL v_thread_id;
END IF;
SET _txns_count = _txns_count + 1;
END IF;
UNTIL done END REPEAT;
IF _txns_count < 1 THEN
SET _txns_killed = 'None';
END IF;
SELECT _txns_killed;
END
$$
DELIMITER ;
DROP EVENT IF EXISTS kill_long_running_txns;
"""
stdout, stderr = self._zendb('mysql', check_sql)
try:
if int(stdout) > 0:
if not silent:
LOG.info(
"Database already has mysql.KillTransactions stored procedure. No action required."
)
else:
stdout, stderr = self._zendb('mysql', sql)
if not silent:
LOG.info(
"Created mysql.KillTransactions stored procedure.")
except ValueError:
if not silent:
LOG.error(
"Unable to determine if mysql.KillTransactions stored procedure exists in database!"
)
exit(1)
backend = getConfigOption(zenPath('etc', configFile), 'ping-backend',
backend)
return backend
if __name__ == '__main__':
# load zcml for the product
import Products.ZenossStartup
from Zope2.App import zcml
zcml.load_site()
pingBackend = getPingBackend()
myPreferences = zope.component.getUtility(
ZenStatus.interfaces.IPingCollectionPreferences, pingBackend)
myTaskFactory = zope.component.getUtility(
ZenStatus.interfaces.IPingTaskFactory, pingBackend)
myTaskSplitter = PerIpAddressTaskSplitter(myTaskFactory)
myDaemon = PingDaemon(
myPreferences,
myTaskSplitter,
stoppingCallback=myPreferences.preShutdown,
)
# add trace cache to preferences, so tasks can find it
traceCachePath = zenPath('var', 'zenping', myDaemon.options.monitor)
myPreferences.options.traceCache = FileCache(traceCachePath)
myDaemon.run()
def _zCommandPath(self, dmd):
from Products.ZenUtils.Utils import zenPath
dmd.Devices.zCommandPath = zenPath('libexec')
def path(self, *parts):
return zenPath('Products', self.id, *parts)
def show_daemon_xml_configs(self, daemon, REQUEST=None):
"""
Display the daemon configuration options in an XML format.
Merges the defaults with options in the config file.
"""
# Sanity check
if not daemon or daemon == '':
messaging.IMessageSender(self).sendToBrowser(
'Internal Error',
'Called without a daemon name',
priority=messaging.WARNING)
return []
if daemon in ['zeoctl', 'zopectl', 'zeneventserver']:
return []
# sanitize the input
if daemon not in self._getDaemonList():
return []
if not self.isValidDaemon(daemon):
messaging.IMessageSender(self).sendToBrowser(
'Internal Error',
'%s is not a valid daemon name' % daemon,
priority=messaging.WARNING)
return []
xml_default_name = zenPath("etc", daemon + ".xml")
try:
# Always recreate the defaults file in order to avoid caching issues
log.debug("Creating XML config file for %s" % daemon)
make_xml = ' '.join(
[binPath(daemon), "genxmlconfigs", ">", xml_default_name])
proc = Popen(make_xml, shell=True, stdout=PIPE, stderr=PIPE)
output, errors = proc.communicate()
proc.wait()
if proc.returncode != 0:
log.error(errors)
messaging.IMessageSender(self).sendToBrowser(
'Internal Error', errors, priority=messaging.CRITICAL)
return [["Output", output, errors, make_xml, "string"]]
except Exception as ex:
msg = "Unable to execute '%s'\noutput='%s'\nerrors='%s'\nex=%s" % (
make_xml, output, errors, ex)
log.error(msg)
messaging.IMessageSender(self).sendToBrowser(
'Internal Error', msg, priority=messaging.CRITICAL)
return [["Error in command", output, errors, make_xml, "string"]]
try:
xml_defaults = parse(xml_default_name)
except Exception:
info = traceback.format_exc()
msg = "Unable to parse XML file %s because %s" % (xml_default_name,
info)
log.error(msg)
messaging.IMessageSender(self).sendToBrowser(
'Internal Error', msg, priority=messaging.CRITICAL)
return [[
"Error parsing XML file", xml_default_name, "XML", info,
"string"
]]
configfile = self._getConfigFilename(daemon)
try:
# Grab the current configs
current_configs = self.parseconfig(configfile)
except Exception:
info = traceback.format_exc()
msg = "Unable to obtain current configuration from %s because %s" % (
configfile, info)
log.error(msg)
messaging.IMessageSender(self).sendToBrowser(
'Internal Error', msg, priority=messaging.CRITICAL)
return [[
"Configuration file issue", configfile, configfile, info,
"string"
]]
all_options = {}
ignore_options = ['configfile', 'cycle', 'daemon', 'weblog']
try:
for option in xml_defaults.getElementsByTagName('option'):
id = option.attributes['id'].nodeValue
if id in ignore_options:
continue
try:
help = unquote(option.attributes['help'].nodeValue)
except Exception:
help = ''
try:
default = unquote(option.attributes['default'].nodeValue)
except Exception:
default = ''
if default == '[]': # Attempt at a list argument -- ignore
continue
all_options[id] = [
id,
current_configs.get(id, default),
default,
help.replace("%default", default),
option.attributes['type'].nodeValue,
]
except Exception:
info = traceback.format_exc()
msg = "Unable to merge XML defaults with config file" \
" %s because %s" % (configfile, info)
log.error(msg)
messaging.IMessageSender(self).sendToBrowser(
'Internal Error', msg, priority=messaging.CRITICAL)
return [[
"XML file issue", daemon, xml_default_name, info, "string"
]]
return [all_options[name] for name in sorted(all_options.keys())]
def cutover(self, unused):
for directory in ['import', 'export']:
path = zenPath(directory)
if not os.path.exists(path):
os.mkdir(path, 0750)
