def migrate(self, pack):
# Apply only on ZP upgrade.
if pack.prevZenPackVersion is None:
return
try:
import servicemigration as sm
except ImportError:
# No servicemigrations, which means we are on Zenoss 4.2.x or 5.0.x
# No need to install service on Zenoss 5.0.x as service install
# performed on each upgrade.
return
sm.require("1.0.0")
try:
ctx = sm.ServiceContext()
except:
log.warn("Couldn't generate service context, skipping.")
return
# Check whether zenmapper service is already installed.
services = filter(lambda s: s.name == "zenmapper", ctx.services)
if not services:
log.info('Deploying zenmapper service')
pack.installServices()
def migrate(self, pack):
if pack.prevZenPackVersion is None:
# Do nothing if this is a fresh install of self.version.
return
if not CONTROL_CENTER:
return
sm.require("1.0.0")
try:
ctx = sm.ServiceContext()
except Exception as e:
LOG.warn("failed to replace %s service: %s", SERVICE_NAME, e)
return
service = get_service_id(ctx)
if not service:
return
client = ControlPlaneClient(**getConnectionSettings())
# Stop and remove the old service.
remove_service(client, service)
# Install the new service.
pack.installServices()
def migrate(self, dmd):
log.info("Add volume for zenjmx-libs")
try:
import servicemigration as sm
sm.require("1.1.5")
except ImportError:
log.debug("Couldn't import servicemigration, skipping")
return
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.debug("Couldn't generate service context, skipping")
return
volume_exist = False
created = False
for inst in (s for s in ctx.services if 'zenjmx' in s.name):
for volume in inst.volumes:
if volume.containerPath == zenjmx_libs_containerPath:
volume_exist = True
log.debug("Volume for zenjmx-libs already exists")
break
if not volume_exist:
inst.volumes.append(
sm.volume.Volume(containerPath=zenjmx_libs_containerPath,
owner=zenjmx_libs_owner,
permission=zenjmx_libs_permission,
resourcePath=zenjmx_libs_resourcePath))
created = True
if created:
log.info("Volume for zenjmx-libs created")
ctx.commit()
def test_sdk_versioning_bugfix_big(self):
major = int(sm.version.API_VERSION.split('.')[0])
minor = int(sm.version.API_VERSION.split('.')[1])
bugfx = int(sm.version.API_VERSION.split('.')[2]) + 1
try:
sm.require("%d.%d.%d" % (major, minor, bugfx))
except ValueError:
pass
else:
raise ValueError("SDK Versioning logic failed.")
def test_sdk_versioning_bugfix_big(self):
major = int(sm.version.API_VERSION.split('.')[0])
minor = int(sm.version.API_VERSION.split('.')[1])
bugfx = int(sm.version.API_VERSION.split('.')[2]) + 1
try:
sm.require("%d.%d.%d" % (major, minor, bugfx))
except ValueError:
pass
else:
raise ValueError("SDK Versioning logic failed.")
def migrate(self, pack):
# Apply only on ZP upgrade.
if pack.prevZenPackVersion is None:
return
try:
import servicemigration as sm
from servicemigration.endpoint import Endpoint
except ImportError:
# No servicemigrations, which means we are on Zenoss 4.2.x or 5.0.x
# No need to install service on Zenoss 5.0.x as service install
# performed on each upgrade.
return
sm.require("1.0.0")
try:
ctx = sm.ServiceContext()
except:
log.warn("Couldn't generate service context, skipping.")
return
redis_endpoint = Endpoint(name="redis",
purpose="import",
application="redis",
portnumber=6379,
protocol="tcp")
log.info("Looking for zenmapper services to migrate")
services = filter(lambda s: s.name == "zenmapper", ctx.services)
# Add the Redis endpoint import if it does not exist
if not services:
log.info("Found no 'zenmapper' services to migrate")
# short circuit
return
for service in services:
if not filter(
lambda endpoint: endpoint.purpose == "import" and endpoint.
application == 'redis', service.endpoints):
log.info("Adding 'redis' endpoint import to service '%s'",
service.name)
service.endpoints.append(redis_endpoint)
ctx.commit()
def migrate(self, pack):
# Apply only on ZP upgrade.
if pack.prevZenPackVersion is None:
return
try:
import servicemigration as sm
except ImportError:
# No servicemigrations, which means we are on Zenoss 4.2.x or 5.0.x
# No need to install service on Zenoss 5.0.x as service install
# performed on each upgrade.
return
sm.require("1.0.0")
try:
ctx = sm.ServiceContext()
except:
log.warn("Couldn't generate service context, skipping.")
return
log.info("Looking for zenmapper services to migrate")
services = filter(lambda s: s.name == "zenmapper", ctx.services)
# Add the Redis endpoint import if it does not exist
if not services:
log.info("Found no 'zenmapper' services to migrate")
# short circuit
return
for service in services:
try:
if not service.ramCommitment:
service.ramCommitment = '5G'
except AttributeError:
log.warning("Your Zenoss version doesn't support such "
"migrations. Please set RAM commitment for "
"zenmapper service manualy to 5G using "
"Control Center UI.")
return
ctx.commit()
def migrate(self, pack):
# Apply only on ZP upgrade.
if pack.prevZenPackVersion is None:
return
try:
import servicemigration as sm
except ImportError:
# No servicemigrations, which means we are on Zenoss 4.2.x or 5.0.x
# No need to install service on Zenoss 5.0.x as service install
# performed on each upgrade.
return
sm.require("1.0.0")
try:
ctx = sm.ServiceContext()
except:
log.warn("Couldn't generate service context, skipping.")
return
log.info("Looking for zenmapper services to migrate")
services = filter(lambda s: s.name == "zenmapper", ctx.services)
# Add the Redis endpoint import if it does not exist
if not services:
log.info("Found no 'zenmapper' services to migrate")
# short circuit
return
for service in services:
try:
if not service.ramCommitment:
service.ramCommitment = '5G'
except AttributeError:
log.warning("Your Zenoss version doesn't support such "
"migrations. Please set RAM commitment for "
"zenmapper service manualy to 5G using "
"Control Center UI.")
return
ctx.commit()
##############################################################################
#
# Copyright (C) Zenoss, Inc. 2016, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import Migrate
import servicemigration as sm
sm.require("1.1.7")
class AddAuditLogFlag(Migrate.Step):
""" Add boolean flag to indicate which logs are audit logs."""
version = Migrate.Version(114, 0, 0)
def cutover(self, dmd):
log = logging.getLogger("zen.migrate")
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
changed = False
__doc__ = """AddAPIZopesSvcDef
Adds the service zenapi that provides Zope instances dedicated to serving non-UI JSON API requests.
"""
import logging
log = logging.getLogger("zen.migrate")
import os
import re
import copy
import Migrate
import servicemigration as sm
import subprocess
import shutil
sm.require("1.1.9")
def create_upstream_pattern(upstream_name, server_decl):
return re.compile("\\n\s+upstream %s {\\n\s+least_conn;\\n\s+%s;\\n\s+keepalive 64;\\n\s+}(?:(?:\\r)?\\n)" % (upstream_name, server_decl))
incl_mimetypes_pat = re.compile(r'include mime\.types;(?:(?:\\r)?\\n)?')
zope_upstream_pat = create_upstream_pattern('zopes', 'include zope-upstreams\.conf')
zopereports_upstream_pat = create_upstream_pattern('zopereports', 'include zopereports-upstreams\.conf')
debugzope_upstream_pat = create_upstream_pattern('debugzopes', 'server 127\.0\.0\.1:9310')
apizopes_upstream_pat = create_upstream_pattern('apizopes', 'include apizopes-upstreams\.conf')
map_whichzopes_pat = re.compile(r'\\n\s+map \$host \$whichzopes {\\n\s+default zopes;\\n\s+~\*zenapi apizopes;\\n\s+}\\n(?:(?:\\r)?\\n)')
apizopes_upstreams_decl = '\n\n upstream apizopes {\n least_conn;\n include apizopes-upstreams.conf;\n keepalive 64;\n }\n'
apizopes_map_clause = '\n ~*zenapi apizopes;'
apizopes_map_whichzopes_block_decl = '\n map $host $whichzopes {\n default zopes;\n ~*zenapi apizopes;\n }\n'
#
# Copyright (C) Zenoss, Inc. 2017, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import Migrate
import os
import servicemigration as sm
from Products.ZenUtils.Utils import zenPath
log = logging.getLogger("zen.migrate")
sm.require("1.1.10")
class AddZookeeperLogFilters(Migrate.Step):
"""
Add LogFilters for Zookeeper logs issue addressed by ZEN-28089
"""
version = Migrate.Version(200, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
service = ctx.getTopService()
import os
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
from servicemigration.metrics import Metric
from servicemigration.graphrange import GraphRange
from servicemigration.graphconfig import GraphConfig
from servicemigration.metricconfig import MetricConfig
from servicemigration.graphdatapoint import GraphDatapoint
sm.require("1.1.12")
class AddZingConfigToZep(Migrate.Step):
"Add reasonable defaults to zeneventserver to enable sending events to Zenoss cloud"
version = Migrate.Version(300, 0, 6)
CLOUD_CONFIG_BLOCK = """
## Configuration to forward events to Zenoss Cloud
#
zep.zing.enabled=false
zep.zing.project={{getContext . "cse.project"}}
zep.zing.tenant={{getContext . "cse.tenant"}}
zep.zing.source={{getContext . "cse.source"}}
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
sm.require("1.1.8")
zproxy_supervisord_conf = '''\
[supervisord]
nodaemon=true
logfile = /opt/zenoss/log/zproxy_supervisord.log
[unix_http_server]
file=/tmp/supervisor.sock
[supervisorctl]
serverurl=unix:///tmp/supervisor.sock ; use a unix:// URL for a unix socket
[rpcinterface:supervisor]
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
[program:redis]
command=redis-server /etc/redis.conf
autorestart=true
autostart=true
startsecs=5
priority=1
[program:zproxy]
command=/opt/zenoss/zproxy/sbin/zproxy start
# Copyright (C) Zenoss, Inc. 2019, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import Migrate
import servicemigration as sm
from pkg_resources import parse_version
from Products.ZenUtils.controlplane.client import getCCVersion
log = logging.getLogger("zen.migrate")
sm.require("1.1.13")
class AddOOMParams(Migrate.Step):
"Add OOMKillDisable and OOMScoreAdj parameters to db services"
version = Migrate.Version(300, 0, 10)
def cutover(self, dmd):
cc_version = parse_version(getCCVersion())
if cc_version < parse_version("1.6.5"):
log.info("Require CC version >= 1.6.5, skipping")
return
try:
ctx = sm.ServiceContext()
# Copyright (C) Zenoss, Inc. 2015, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import itertools
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
from servicemigration.endpoint import Endpoint
sm.require("1.0.0")
class AddEndpointsToZenjobs(Migrate.Step):
"Add zep + zproxy endpoints to zenjobs services"
version = Migrate.Version(5,0,70)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
endpoint_map = {
import unittest
import os
import servicemigration as sm
sm.require(sm.version.API_VERSION)
INFILENAME = os.path.join(os.path.dirname(__file__), "v1.0.0.json")
OUTFILENAME = os.path.join(os.path.dirname(__file__), "out.json")
class LogConfigTest(unittest.TestCase):
def tests_logconfigs_add(self):
"""
Tests adding a new file to a service's LogConfigs.
"""
ctx = sm.ServiceContext(INFILENAME)
svc = filter(lambda x: x.description == "Zope server", ctx.services)[0]
self.assertEqual(len(svc.logConfigs), 3)
svc.logConfigs.append(
sm.LogConfig(path="/opt/zenoss/log/honk.log", logType="honk"))
ctx.commit(OUTFILENAME)
ctx = sm.ServiceContext(OUTFILENAME)
svc = filter(lambda x: x.description == "Zope server", ctx.services)[0]
if not "honk" in [hc.logType for hc in svc.logConfigs]:
raise ValueError("Failed to alter logconfigs.")
for lc in svc.logConfigs:
if lc.logType == "honk":
self.assertEqual(lc.path, "/opt/zenoss/log/honk.log")
self.assertEqual(lc.isAudit, False)
self.assertEqual(len(svc.logConfigs), 4)
#
# Copyright (C) Zenoss, Inc. 2017, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import Migrate
import os
import servicemigration as sm
from Products.ZenUtils.Utils import zenPath
log = logging.getLogger("zen.migrate")
sm.require("1.1.5")
class updateOpenTSDBOriginalConfig(Migrate.Step):
"""
Overwrite Original OpenTSDB config to match the lastest.
This is needed for the systems that were originaly upgraded from 5.0.5 and lower.
"""
version = Migrate.Version(200, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
import servicemigration.thresholdconfig
import servicemigration.threshold
import servicemigration.eventtags
sm.require("1.1.6")
class addMissedRunsThreshold(Migrate.Step):
"""Adds a threshold for missedRuns for collector services"""
version = Migrate.Version(200, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate context, skipping.")
return
# Get all collector services
##############################################################################
#
# Copyright (C) Zenoss, Inc. 2018, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import servicemigration as sm
import Migrate
log = logging.getLogger("zen.migrate")
sm.require("1.1.14")
class MarkHubServices(Migrate.Step):
"""Update hub services' tags with "hub" marker."""
version = Migrate.Version(300, 0, 11)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
total = 0
zenhubs = (s for s in ctx.services if s.name == "zenhub")
import unittest
import os
import servicemigration as sm
sm.require(sm.version.API_VERSION)
INFILENAME = os.path.join(os.path.dirname(__file__), "v1.0.0.json")
OUTFILENAME = os.path.join(os.path.dirname(__file__), "out.json")
class ServiceTest(unittest.TestCase):
def test_imageId_change(self):
"""
Tests changing an imageID
"""
imageID = "localhost:5000/fake:latest"
ctx = sm.ServiceContext(INFILENAME)
svc = filter(lambda x: x.description == "Zope server", ctx.services)[0]
svc.imageID = imageID
ctx.commit(OUTFILENAME)
ctx = sm.ServiceContext(OUTFILENAME)
svc = filter(lambda x: x.description == "Zope server", ctx.services)[0]
self.assertEqual(svc.imageID, imageID)
def test_description_remove(self):
"""
Tests completely removing a description.
"""
ctx = sm.ServiceContext(INFILENAME)
svc = filter(lambda x: x.description == "Zope server", ctx.services)[0]
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
import servicemigration.thresholdconfig
import servicemigration.threshold
import servicemigration.eventtags
sm.require("1.1.6")
class addMissedRunsThreshold(Migrate.Step):
"""Adds a threshold for missedRuns for collector services"""
version = Migrate.Version(200, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate context, skipping.")
return
# Get all collector services
changed = False
def test_sdk_versioning_bugfix_small(self):
major = int(sm.version.API_VERSION.split('.')[0])
minor = int(sm.version.API_VERSION.split('.')[1])
bugfx = int(sm.version.API_VERSION.split('.')[2]) - 1
sm.require("%d.%d.%d" % (major, minor, bugfx))
#
# Copyright (C) Zenoss, Inc. 2017, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import time
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
sm.require("1.1.11")
class RateCutoff(Migrate.Step):
"""Fix the credentials for consumer and query services"""
version = Migrate.Version(300, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
query_svcs = filter(lambda s: s.name == 'CentralQuery', ctx.services)
def test_sdk_versioning_bugfix_small(self):
major = int(sm.version.API_VERSION.split('.')[0])
minor = int(sm.version.API_VERSION.split('.')[1])
bugfx = int(sm.version.API_VERSION.split('.')[2]) - 1
sm.require("%d.%d.%d" % (major, minor, bugfx))
##############################################################################
#
# Copyright (C) Zenoss, Inc. 2016, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
sm.require("1.1.5")
from serviceRunLevels import serviceRunLevels
class AddServiceRunLevels(Migrate.Step):
"""
Add the emergency shutdown and startup levels to service definitions.
See ZEN-23931, ZEN-27299.
"""
version = Migrate.Version(109, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
##############################################################################
#
# Copyright (C) Zenoss, Inc. 2016, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
import Migrate
import servicemigration as sm
sm.require("1.1.7")
class AddAuditLogFlag(Migrate.Step):
""" Add boolean flag to indicate which logs are audit logs."""
version = Migrate.Version(114, 0, 0)
def cutover(self, dmd):
log = logging.getLogger("zen.migrate")
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
changed = False
for service in ctx.services:
# Copyright (C) Zenoss, Inc. 2016, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import itertools
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
from servicemigration.endpoint import Endpoint
sm.require("1.0.0")
class AddZepEndpointToZeneventd(Migrate.Step):
"Add zep endpoint to zeneventd services"
version = Migrate.Version(5,0,70)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
endpoint_map = {
#
# Copyright (C) Zenoss, Inc. 2017, all rights reserved.
#
# This content is made available according to terms specified in
# License.zenoss under the directory where your Zenoss product is installed.
#
##############################################################################
import logging
log = logging.getLogger("zen.migrate")
import Migrate
import servicemigration as sm
sm.require("1.1.11")
class DeleteRateOptionsForServices(Migrate.Step):
"""
Update/Add rateOptions for counters.
Remove reset values.
"""
version = Migrate.Version(300, 0, 1)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
#
##############################################################################
import os
import re
import logging
log = logging.getLogger("zen.migrate")
from Products.ZenUtils.Utils import zenPath
import Migrate
import servicemigration as sm
from servicemigration import InstanceLimits
sm.require("1.1.10")
class FixHbaseLogFilters(Migrate.Step):
"""Correct the LogFilters for HBase (ZEN-28084)"""
version = Migrate.Version(200, 0, 0)
def cutover(self, dmd):
try:
ctx = sm.ServiceContext()
except sm.ServiceMigrationError:
log.info("Couldn't generate service context, skipping.")
return
__doc__ = """AddAPIZopesSvcDef
Adds the service zenapi that provides Zope instances dedicated to serving non-UI JSON API requests.
"""
import logging
log = logging.getLogger("zen.migrate")
import os
import re
import copy
import Migrate
import servicemigration as sm
import subprocess
import shutil
sm.require("1.1.9")
def create_upstream_pattern(upstream_name, server_decl):
return re.compile("\\n\s+upstream %s {\\n\s+least_conn;\\n\s+%s;\\n\s+keepalive 64;\\n\s+}(?:(?:\\r)?\\n)" % (upstream_name, server_decl))
incl_mimetypes_pat = re.compile(r'include mime\.types;(?:(?:\\r)?\\n)?')
zope_upstream_pat = create_upstream_pattern('zopes', 'include zope-upstreams\.conf')
zopereports_upstream_pat = create_upstream_pattern('zopereports', 'include zopereports-upstreams\.conf')
debugzope_upstream_pat = create_upstream_pattern('debugzopes', 'server 127\.0\.0\.1:9310')
apizopes_upstream_pat = create_upstream_pattern('apizopes', 'include apizopes-upstreams\.conf')
map_whichzopes_pat = re.compile(r'\\n\s+map \$host \$whichzopes {\\n\s+default zopes;\\n\s+~\*zenapi apizopes;\\n\s+}\\n(?:(?:\\r)?\\n)')
apizopes_upstreams_decl = '\n\n upstream apizopes {\n least_conn;\n include apizopes-upstreams.conf;\n keepalive 64;\n }\n'
apizopes_map_clause = '\n ~*zenapi apizopes;'
apizopes_map_whichzopes_block_decl = '\n map $host $whichzopes {\n default zopes;\n ~*zenapi apizopes;\n }\n'
