def discover_network_types(dbapi_con, connection_record): # pylint: disable=W0613
config = Config()
if not config.has_option("broker", "default_network_type"): # pragma: no cover
raise InternalError("The default_network_type option is missing from "
"the [broker] section in the configuration.")
default_type = config.get("broker", "default_network_type")
default_section = "network_" + default_type
if not config.has_section(default_section): # pragma: no cover
raise InternalError("The default network type is %s, but there's no "
"section named [%s] in the configuration." %
(default_type, default_section))
nettypes = {}
# This function should be called only once, but you never know...
if Network.network_type_map:
return
for section in config.sections():
if not section.startswith("network_"):
continue
name = section[8:]
nettypes[name] = NetworkProperties(config, name)
LOGGER.info("Configured network type %s" % name)
Network.network_type_map = nettypes
Network.default_network_props = nettypes[default_type]
def onEnter(self, dbcluster):
dbdecommissioned = HostLifecycle.get_unique(object_session(dbcluster),
"decommissioned",
compel=True)
config = Config()
archetype = dbcluster.personality.archetype
section = "archetype_" + archetype.name
opt = "allow_cascaded_deco"
if dbcluster.hosts and (not config.has_option(section, opt)
or not config.getboolean(section, opt)):
raise ArgumentError("Cannot change state to {0}, as {1}'s "
"archetype is {2}.".format(
dbdecommissioned.name, dbcluster,
archetype.name))
if dbcluster.machines:
raise ArgumentError("Cannot change state to {0}, as {1} has "
"{2} VM(s).".format(dbdecommissioned.name,
dbcluster,
len(dbcluster.machines)))
for dbhost in dbcluster.hosts:
dbhost.status.transition(dbhost, dbdecommissioned)
def discover_network_types(dbapi_con, connection_record): # pylint: disable=W0613
config = Config()
if not config.has_option("broker", "default_network_type"): # pragma: no cover
raise InternalError("The default_network_type option is missing from "
"the [broker] section in the configuration.")
default_type = config.get("broker", "default_network_type")
default_section = "network_" + default_type
if not config.has_section(default_section): # pragma: no cover
raise InternalError("The default network type is %s, but there's no "
"section named [%s] in the configuration." %
(default_type, default_section))
nettypes = {}
# This function should be called only once, but you never know...
if Network.network_type_map:
return
for section in config.sections():
if not section.startswith("network_"):
continue
name = section[8:]
nettypes[name] = NetworkProperties(config, name)
LOGGER.info("Configured network type %s", name)
Network.network_type_map = nettypes
Network.default_network_props = nettypes[default_type]
def run_git(args, env=None, path=".",
logger=LOGGER, loglevel=logging.INFO, filterre=None):
config = Config()
if env:
git_env = env.copy()
else:
git_env = {}
env_path = git_env.get("PATH", os.environ.get("PATH", ""))
git_env["PATH"] = "%s:%s" % (config.get("broker", "git_path"), env_path)
for name in ["git_author_name", "git_author_email",
"git_committer_name", "git_committer_email"]:
if not config.has_option("broker", name):
continue
value = config.get("broker", name)
git_env[name.upper()] = value
if isinstance(args, list):
git_args = args[:]
if git_args[0] != "git":
git_args.insert(0, "git")
else:
git_args = ["git", args]
return run_command(git_args, env=git_env, path=path,
logger=logger, loglevel=loglevel, filterre=filterre)
def onEnter(self, dbcluster):
dbdecommissioned = HostLifecycle.get_unique(object_session(dbcluster),
"decommissioned",
compel=True)
config = Config()
archetype = dbcluster.personality.archetype
section = "archetype_" + archetype.name
opt = "allow_cascaded_deco"
if dbcluster.hosts and (not config.has_option(section, opt) or
not config.getboolean(section, opt)):
raise ArgumentError("Cannot change state to {0}, as {1}'s "
"archetype is {2}."
.format(dbdecommissioned.name, dbcluster,
archetype.name))
if dbcluster.virtual_machines:
raise ArgumentError("Cannot change state to {0}, as {1} has "
"{2} VM(s)."
.format(dbdecommissioned.name, dbcluster,
len(dbcluster.virtual_machines)))
for dbhost in dbcluster.hosts:
dbhost.status.transition(dbhost, dbdecommissioned)
def testclonetemplateking(self):
config = Config()
source = config.get("unittest", "template_base")
dest = config.get("broker", "kingdir")
p = Popen(("/bin/rm", "-rf", dest), stdout=1, stderr=2)
rc = p.wait()
self.assertEqual(
rc, 0, "Failed to clear old template-king directory '%s'" % dest)
env = {}
env["PATH"] = "%s:%s" % (config.get(
"broker", "git_path"), os.environ.get("PATH", ""))
p = Popen(("git", "clone", "--bare", source, dest),
env=env,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code for clone of template-king, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (out, err))
# This value can be used to test against a different branch/commit
# than the current 'prod'.
new_prod = None
if config.has_option("unittest", "template_alternate_prod"):
new_prod = config.get("unittest", "template_alternate_prod")
if new_prod:
for domain in ['prod', 'ny-prod']:
p = Popen(("git", "push", ".", '+%s:%s' % (new_prod, domain)),
env=env,
cwd=dest,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code while setting alternate "
"'%s' branch locally to '%s':"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(domain, new_prod, out, err))
# Set the default branch
p = Popen(("git", "symbolic-ref", "HEAD", "refs/heads/prod"),
env=env,
cwd=dest,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0, "Non-zero return code while setting HEAD "
"to refs/heads/prod:"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n" % (out, err))
return
def teststart(self):
# FIXME: Either remove any old pidfiles, or ignore it as a warning
# from stderr... or IMHO (daqscott) if pid files exist and are knc or
# python processes, kill -9 the pids and delete the files (with a
# warning message it tickles you)
config = Config()
twistd = os.path.join(config.get("broker", "srcdir"), "lib",
"python2.6", "aquilon", "unittest_patches.py")
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
logfile = config.get("broker", "logfile")
# Specify twistd and options...
args = [
sys.executable, twistd, "--pidfile", pidfile, "--logfile", logfile
]
if config.has_option("unittest", "profile"):
if config.getboolean("unittest", "profile"):
args.append("--profile")
args.append(
os.path.join(config.get("broker", "logdir"),
"aqd.profile"))
args.append("--profiler=cProfile")
args.append("--savestats")
# And then aqd and options...
args.extend(["aqd", "--config", config.baseconfig])
if config.has_option("unittest", "coverage"):
if config.getboolean("unittest", "coverage"):
args.append("--coveragedir")
dir = os.path.join(config.get("broker", "logdir"), "coverage")
args.append(dir)
coveragerc = os.path.join(config.get("broker", "srcdir"),
"tests", "coverage.rc")
args.append("--coveragerc")
args.append(coveragerc)
p = Popen(args)
self.assertEqual(p.wait(), 0)
def teststart(self):
# FIXME: Either remove any old pidfiles, or ignore it as a warning
# from stderr... or IMHO (daqscott) if pid files exist and are knc or
# python processes, kill -9 the pids and delete the files (with a
# warning message it tickles you)
config = Config()
twistd = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon", "unittest_patches.py")
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
logfile = config.get("broker", "logfile")
# Specify twistd and options...
args = [sys.executable, twistd,
"--pidfile", pidfile, "--logfile", logfile]
if config.has_option("unittest", "profile"):
if config.getboolean("unittest", "profile"):
args.append("--profile")
args.append(os.path.join(config.get("broker", "logdir"),
"aqd.profile"))
args.append("--profiler=cProfile")
args.append("--savestats")
# And then aqd and options...
args.extend(["aqd", "--config", config.baseconfig])
if config.has_option("unittest", "coverage"):
if config.getboolean("unittest", "coverage"):
args.append("--coveragedir")
dir = os.path.join(config.get("broker", "logdir"), "coverage")
args.append(dir)
coveragerc = os.path.join(config.get("broker", "srcdir"),
"tests", "coverage.rc")
args.append("--coveragerc")
args.append(coveragerc)
p = Popen(args)
self.assertEqual(p.wait(), 0)
def testclonetemplateking(self):
config = Config()
source = config.get("unittest", "template_base")
dest = config.get("broker", "kingdir")
p = Popen(("/bin/rm", "-rf", dest), stdout=1, stderr=2)
rc = p.wait()
self.assertEqual(rc, 0,
"Failed to clear old template-king directory '%s'" %
dest)
env = {}
env["PATH"] = "%s:%s" % (config.get("broker", "git_path"),
os.environ.get("PATH", ""))
p = Popen(("git", "clone", "--bare", source, dest),
env=env, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code for clone of template-king, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (out, err))
# This value can be used to test against a different branch/commit
# than the current 'prod'.
new_prod = None
if config.has_option("unittest", "template_alternate_prod"):
new_prod = config.get("unittest", "template_alternate_prod")
if new_prod:
for domain in ['prod', 'ny-prod']:
p = Popen(("git", "push", ".", '+%s:%s' % (new_prod, domain)),
env=env, cwd=dest, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code while setting alternate "
"'%s' branch locally to '%s':"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (domain, new_prod, out, err))
# Set the default branch
p = Popen(("git", "symbolic-ref", "HEAD", "refs/heads/prod"),
env=env, cwd=dest, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Non-zero return code while setting HEAD "
"to refs/heads/prod:"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (out, err))
return
def main():
from aquilon.config import Config
config = Config()
if config.has_option("database", "module"):
ms.modulecmd.load(config.get("database", "module"))
db = DbFactory()
Base.metadata.bind = db.engine
session = db.Session()
add_interfaces(session)
add_addresses(session)
session.rollback()
raise Exception("Replace the rollback() in the code with commit() when "
"ready to go, and disable this exception")
def main():
from aquilon.config import Config
config = Config()
if config.has_option("database", "module"):
ms.modulecmd.load(config.get("database", "module"))
db = DbFactory()
Base.metadata.bind = db.engine
session = db.Session()
add_interfaces(session)
add_addresses(session)
session.rollback()
raise Exception("Replace the rollback() in the code with commit() when "
"ready to go, and disable this exception")
def run_git(args,
env=None,
path=".",
logger=LOGGER,
loglevel=logging.INFO,
filterre=None):
config = Config()
if env:
git_env = env.copy()
else:
git_env = {}
env_path = git_env.get("PATH", os.environ.get("PATH", ""))
git_env["PATH"] = "%s:%s" % (config.get("broker", "git_path"), env_path)
for name in [
"git_author_name", "git_author_email", "git_committer_name",
"git_committer_email"
]:
if not config.has_option("broker", name):
continue
value = config.get("broker", name)
git_env[name.upper()] = value
if isinstance(args, list):
git_args = args[:]
if git_args[0] != "git":
git_args.insert(0, "git")
else:
git_args = ["git", args]
return run_command(git_args,
env=git_env,
path=path,
logger=logger,
loglevel=loglevel,
filterre=filterre)
def makeService(self, options):
# Start up coverage ASAP.
coverage_dir = options["coveragedir"]
if coverage_dir:
os.makedirs(coverage_dir, 0755)
if options["coveragerc"]:
coveragerc = options["coveragerc"]
else:
coveragerc = None
self.coverage = coverage.coverage(config_file=coveragerc)
self.coverage.erase()
self.coverage.start()
# Get the config object.
config = Config(configfile=options["config"])
# Helper for finishing off the coverage report.
def stop_coverage():
log.msg("Finishing coverage")
self.coverage.stop()
aquilon_srcdir = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon")
sourcefiles = []
for dirpath, dirnames, filenames in os.walk(aquilon_srcdir):
# FIXME: try to do this from the coverage config file
if dirpath.endswith("aquilon"):
dirnames.remove("client")
elif dirpath.endswith("aqdb"):
dirnames.remove("utils")
for filename in filenames:
if not filename.endswith('.py'):
continue
sourcefiles.append(os.path.join(dirpath, filename))
self.coverage.html_report(sourcefiles, directory=coverage_dir)
self.coverage.xml_report(sourcefiles,
outfile=os.path.join(
coverage_dir, "aqd.xml"))
with open(os.path.join(coverage_dir, "aqd.coverage"),
"w") as outfile:
self.coverage.report(sourcefiles, file=outfile)
# Make sure the coverage report gets generated.
if coverage_dir:
reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage)
# Set up the environment...
m = Modulecmd()
log_module_load(m, config.get("broker", "CheckNet_module"))
if config.has_option("database", "module"):
log_module_load(m, config.get("database", "module"))
sys.path.append(config.get("protocols", "directory"))
# Set this up before the aqdb libs get imported...
integrate_logging(config)
progname = os.path.split(sys.argv[0])[1]
if progname == 'aqd':
if config.get('broker', 'mode') != 'readwrite':
log.msg("Broker started with aqd symlink, "
"setting config mode to readwrite")
config.set('broker', 'mode', 'readwrite')
if progname == 'aqd_readonly':
if config.get('broker', 'mode') != 'readonly':
log.msg("Broker started with aqd_readonly symlink, "
"setting config mode to readonly")
config.set('broker', 'mode', 'readonly')
log.msg("Loading broker in mode %s" % config.get('broker', 'mode'))
# Dynamic import means that we can parse config options before
# importing aqdb. This is a hack until aqdb can be imported without
# firing up database connections.
resources = __import__("aquilon.worker.resources", globals(), locals(),
["RestServer"], -1)
RestServer = getattr(resources, "RestServer")
restServer = RestServer(config)
openSite = AnonSite(restServer)
# twisted is nicely changing the umask for us when the process is
# set to daemonize. This sets it back.
restServer.set_umask()
reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask)
reactor.addSystemEventTrigger('after', 'startup',
restServer.set_thread_pool_size)
sockdir = config.get("broker", "sockdir")
if not os.path.exists(sockdir):
os.makedirs(sockdir, 0700)
os.chmod(sockdir, 0700)
if options["usesock"]:
return strports.service("unix:%s/aqdsock" % sockdir, openSite)
openport = config.get("broker", "openport")
if config.has_option("broker", "bind_address"):
bind_address = config.get("broker", "bind_address").strip()
openaddr = "tcp:%s:interface=%s" % (openport, bind_address)
else: # pragma: no cover
bind_address = None
openaddr = "tcp:%s" % openport
# Return before firing up knc.
if options["noauth"]:
return strports.service(openaddr, openSite)
sockname = os.path.join(sockdir, "kncsock")
# This flag controls whether or not this process will start up
# and monitor knc. Except for noauth mode knc has to be running,
# but this process doesn't have to be the thing that starts it up.
if config.getboolean("broker", "run_knc") or \
config.getboolean("broker", "run_git_daemon"):
mon = GracefulProcessMonitor()
# FIXME: Should probably run krb5_keytab here as well.
# and/or verify that the keytab file exists.
if config.getboolean("broker", "run_knc"):
keytab = config.get("broker", "keytab")
knc_args = [
"/usr/bin/env",
"KRB5_KTNAME=FILE:%s" % keytab,
config.get("kerberos", "knc"), "-lS", sockname
]
if bind_address:
knc_args.append("-a")
knc_args.append(bind_address)
knc_args.append(config.get("broker", "kncport"))
mon.addProcess("knc", knc_args)
if config.getboolean("broker", "run_git_daemon"):
# The git daemon *must* be invoked using the form 'git-daemon'
# instead of invoking git with a 'daemon' argument. The latter
# will fork and exec git-daemon, resulting in a new pid that
# the process monitor won't know about!
gitpath = config.get("broker", "git_path")
gitdaemon = config.get("broker", "git_daemon")
ospath = os.environ.get("PATH", "")
args = [
"/usr/bin/env",
"PATH=%s:%s" % (gitpath, ospath), gitdaemon,
"--export-all",
"--base-path=%s" %
config.get("broker", "git_daemon_basedir")
]
if config.has_option("broker", "git_port"):
args.append("--port=%s" % config.get("broker", "git_port"))
if bind_address:
args.append("--listen=%s" % bind_address)
args.append(config.get("broker", "kingdir"))
mon.addProcess("git-daemon", args)
mon.startService()
reactor.addSystemEventTrigger('before', 'shutdown',
mon.stopService)
# This socket is created by twisted and only accessed by knc as
# connections come in.
if os.path.exists(sockname):
try:
log.msg("Attempting to remove old socket '%s'" % sockname)
os.remove(sockname)
log.msg("Succeeded removing old socket.")
except OSError, e:
log.msg("Could not remove old socket '%s': %s" % (sockname, e))
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.exc import DBAPIError, DatabaseError as SaDBError
from sqlalchemy.schema import CreateIndex
from sqlalchemy.dialects.oracle.base import OracleDDLCompiler
import ms.modulecmd as modcmd
try:
config = Config()
except Exception, e:
print >> sys.stderr, 'failed to read configuration: %s' % e
sys.exit(os.EX_CONFIG)
assert config, 'No configuration in db_factory'
if config.has_option("database", "module"):
modcmd.load(config.get("database", "module"))
# Add support for Oracle-specific index extensions
@compiles(CreateIndex, 'oracle')
def visit_create_index(create, compiler, **kw):
index = create.element
compiler._verify_index_table(index)
preparer = compiler.preparer
text = "CREATE "
if index.unique:
text += "UNIQUE "
if index.kwargs.get("oracle_bitmap", False):
text += "BITMAP "
class TestBrokerCommand(unittest.TestCase):
def setUp(self):
self.config = Config()
self.net = DummyNetworks()
# Need to import protocol buffers after we have the config
# object all squared away and we can set the sys.path
# variable appropriately.
# It would be simpler just to change sys.path in runtests.py,
# but this allows for each test to be run individually (without
# the runtests.py wrapper).
protodir = self.config.get("protocols", "directory")
if protodir not in sys.path:
sys.path.append(protodir)
for m in [
'aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2',
'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2',
'aqdparamdefinitions_pb2', 'aqdparameters_pb2'
]:
globals()[m] = __import__(m)
self.user = self.config.get("broker", "user")
self.sandboxdir = os.path.join(
self.config.get("broker", "templatesdir"), self.user)
self.template_extension = self.config.get("panc", "template_extension")
# This method is cumbersome. Should probably develop something
# like unittest.conf.defaults.
if self.config.has_option("unittest", "scratchdir"):
self.scratchdir = self.config.get("unittest", "scratchdir")
if not os.path.exists(self.scratchdir):
os.makedirs(self.scratchdir)
if self.config.has_option("unittest", "aurora_with_node"):
self.aurora_with_node = self.config.get("unittest",
"aurora_with_node")
else:
self.aurora_with_node = "oyidb1622"
if self.config.has_option("unittest", "aurora_without_node"):
self.aurora_without_node = self.config.get("unittest",
"aurora_without_node")
else:
self.aurora_without_node = "pissp1"
self.gzip_profiles = self.config.getboolean("panc", "gzip_output")
self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml"
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
for name in [
DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE,
DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR
]:
path = os.path.join(dsdb_coverage_dir, name)
try:
os.remove(path)
except OSError:
pass
def tearDown(self):
pass
def template_name(self, *template, **args):
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "template_name() called without domain or sandbox")
return os.path.join(dir, *template) + self.template_extension
def plenary_name(self, *template):
dir = self.config.get("broker", "plenarydir")
return os.path.join(dir, *template) + self.template_extension
def find_template(self, *template, **args):
""" Figure out the extension of an existing template """
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "find_template() called without domain or sandbox")
base = os.path.join(dir, *template)
for extension in [".tpl", ".pan"]:
if os.path.exists(base + extension):
return base + extension
self.assert_(0, "template %s does not exist with any extension" % base)
def build_profile_name(self, *template, **args):
base = os.path.join(self.config.get("broker", "builddir"), "domains",
args.get("domain"), "profiles", *template)
return base + self.template_extension
msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n')
def runcommand(self, command, auth=True, **kwargs):
aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py")
if auth:
port = self.config.get("broker", "kncport")
else:
port = self.config.get("broker", "openport")
if isinstance(command, list):
args = [str(cmd) for cmd in command]
else:
args = [command]
args.insert(0, sys.executable)
args.insert(1, aq)
if "--aqport" not in args:
args.append("--aqport")
args.append(port)
if auth:
args.append("--aqservice")
args.append(self.config.get("broker", "service"))
else:
args.append("--noauth")
if "env" in kwargs:
# Make sure that kerberos tickets are still present if the
# environment is being overridden...
env = {}
for (key, value) in kwargs["env"].items():
env[key] = value
for (key, value) in os.environ.items():
if key.find("KRB") == 0 and key not in env:
env[key] = value
if 'USER' not in env:
env['USER'] = os.environ.get('USER', '')
kwargs["env"] = env
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Strip any msversion dev warnings out of STDERR
err = self.msversion_dev_re.sub('', err)
# Lock messages are pretty common...
err = err.replace(
'Client status messages disabled, '
'retries exceeded.\n', '')
err = LOCK_RE.sub('', err)
return (p, out, err)
def successtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 0, "Non-zero return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err))
return (out, err)
def statustest(self, command, **kwargs):
(out, err) = self.successtest(command, **kwargs)
self.assertEmptyOut(out, command)
return err
def failuretest(self, command, returncode, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, returncode, "Non-%s return code %s for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n" %
(returncode, p.returncode, command, out, err))
return (out, err)
def assertEmptyStream(self, name, contents, command):
self.assertEqual(
contents, "", "%s for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(name, command, contents))
def assertEmptyErr(self, contents, command):
self.assertEmptyStream("STDERR", contents, command)
def assertEmptyOut(self, contents, command):
self.assertEmptyStream("STDOUT", contents, command)
def commandtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEmptyErr(err, command)
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n" %
(command, out))
return out
def noouttest(self, command, **kwargs):
out = self.commandtest(command, **kwargs)
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
def ignoreoutputtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return
# Right now, commands are not implemented consistently. When that is
# addressed, this unit test should be updated.
def notfoundtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
if p.returncode == 0:
self.assertEqual(
err, "", "STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, err))
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
else:
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(
err.find("Not Found") >= 0,
"STDERR for %s did not include Not Found:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
return err
def badrequesttest(self, command, ignoreout=False, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err))
self.failUnless(
err.find("Bad Request") >= 0,
"STDERR for %s did not include Bad Request:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
if not ignoreout and "--debug" not in command:
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs):
(p, out, err) = self.runcommand(command, auth=auth, **kwargs)
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.failUnless(
err.find("Unauthorized:") >= 0,
"STDERR for %s did not include Unauthorized:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
if msgcheck:
self.searchoutput(err, r"Unauthorized (anonymous )?access attempt",
command)
return err
def internalerrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 5, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.assertEqual(
err.find("Internal Server Error"), 0,
"STDERR for %s did not start with "
"Internal Server Error:\n@@@\n'%s'\n@@@\n" % (command, err))
return err
def unimplementederrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 5, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.assertEqual(
err.find("Not Implemented"), 0, "STDERR for %s did not start with "
"Not Implemented:\n@@@\n'%s'\n@@@\n" % (command, err))
return err
# Test for conflicting or invalid aq client options.
def badoptiontest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 2, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 2, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
return err
def partialerrortest(self, command, **kwargs):
# Currently these two cases behave the same way - same exit code
# and behavior.
return self.badoptiontest(command, **kwargs)
def matchoutput(self, out, s, command):
self.assert_(
out.find(s) >= 0,
"output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def matchclean(self, out, s, command):
self.assert_(
out.find(s) < 0, "output for %s includes '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def searchoutput(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failUnless(
m, "output for %s did not match '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
return m
def searchclean(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failIf(
m, "output for %s matches '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
def parse_proto_msg(self, listclass, attr, msg, expect=None):
protolist = listclass()
protolist.ParseFromString(msg)
received = len(getattr(protolist, attr))
if expect is None:
self.failUnless(
received > 0,
"No %s listed in %s protobuf message\n" % (attr, listclass))
else:
self.failUnlessEqual(
received, expect,
"%d %s expected, got %d\n" % (expect, attr, received))
return protolist
def parse_netlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdnetworks_pb2.NetworkList, 'networks',
msg, expect)
def parse_hostlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.HostList, 'hosts', msg,
expect)
def parse_clusters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ClusterList, 'clusters',
msg, expect)
def parse_location_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdlocations_pb2.LocationList, 'locations',
msg, expect)
def parse_dns_domainlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList,
'dns_domains', msg, expect)
def parse_service_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceList, 'services',
msg, expect)
def parse_servicemap_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceMapList,
'servicemaps', msg, expect)
def parse_personality_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.PersonalityList,
'personalities', msg, expect)
def parse_os_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList,
'operating_systems', msg, expect)
def parse_audit_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdaudit_pb2.TransactionList,
'transactions', msg, expect)
def parse_resourcelist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ResourceList, 'resources',
msg, expect)
def parse_paramdefinition_msg(self, msg, expect=None):
return self.parse_proto_msg(
aqdparamdefinitions_pb2.ParamDefinitionList, 'param_definitions',
msg, expect)
def parse_parameters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparameters_pb2.ParameterList,
'parameters', msg, expect)
def gitenv(self, env=None):
"""Configure a known sanitised environment"""
git_path = self.config.get("broker", "git_path")
# The "publish" test abuses gitenv(), and it needs the Python interpreter
# in the path, because it runs the template unit tests which in turn
# call the aq command
python_path = os.path.dirname(sys.executable)
newenv = {}
newenv["USER"] = os.environ.get('USER', '')
if env:
for (key, value) in env.iteritems():
newenv[key] = value
if "PATH" in newenv:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path,
newenv["PATH"])
else:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path,
'/bin:/usr/bin')
return newenv
def gitcommand_raw(self, command, **kwargs):
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, "git")
env = self.gitenv(kwargs.pop("env", None))
p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
return p
def gitcommand(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def gitcommand_expectfailure(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.failIfEqual(
p.returncode, 0,
"Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def check_git_merge_health(self, repo):
command = "merge HEAD"
out = self.gitcommand(command.split(" "), cwd=repo)
return
def grepcommand(self, command, **kwargs):
if self.config.has_option("unittest", "grep"):
grep = self.config.get("unittest", "grep")
else:
grep = "/bin/grep"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, grep)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
if p.returncode == 1:
return []
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(command, out, err))
def findcommand(self, command, **kwargs):
if self.config.has_option("unittest", "find"):
find = self.config.get("unittest", "find")
else:
find = "/usr/bin/find"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, find)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(command, out, err))
def writescratch(self, filename, contents):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'w') as f:
f.write(contents)
return scratchfile
def readscratch(self, filename):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'r') as f:
contents = f.read()
return contents
def dsdb_expect(self, command, fail=False, errstr=""):
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
if fail:
filename = DSDB_EXPECT_FAILURE_FILE
else:
filename = DSDB_EXPECT_SUCCESS_FILE
expected_name = os.path.join(dsdb_coverage_dir, filename)
with open(expected_name, "a") as fp:
if isinstance(command, list):
fp.write(" ".join([str(cmd) for cmd in command]))
else:
fp.write(str(command))
fp.write("\n")
if fail and errstr:
errfile = DSDB_EXPECT_FAILURE_ERROR
expected_name = os.path.join(dsdb_coverage_dir, errfile)
with open(expected_name, "a") as fp:
fp.write(errstr)
fp.write("\n")
def dsdb_expect_add(self,
hostname,
ip,
interface=None,
mac=None,
primary=None,
comments=None,
fail=False):
command = [
"add_host", "-host_name", hostname, "-ip_address",
str(ip), "-status", "aq"
]
if interface:
command.extend(
["-interface_name",
str(interface).replace('/', '_')])
if mac:
command.extend(["-ethernet_address", str(mac)])
if primary:
command.extend(["-primary_host_name", primary])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_delete(self, ip, fail=False):
self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail)
def dsdb_expect_update(self,
fqdn,
iface=None,
ip=None,
mac=None,
comments=None,
fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if iface:
command.extend(["-interface_name", iface])
if ip:
command.extend(["-ip_address", str(ip)])
if mac:
command.extend(["-ethernet_address", str(mac)])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_rename(self,
fqdn,
new_fqdn=None,
iface=None,
new_iface=None,
fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if new_fqdn:
command.extend(["-primary_host_name", new_fqdn])
if iface:
command.extend(["-interface_name", iface])
if new_iface:
command.extend(["-new_interface_name", new_iface])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_add_campus(self,
campus,
comments=None,
fail=False,
errstr=""):
command = ["add_campus_aq", "-campus_name", campus]
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus(self, campus, fail=False, errstr=""):
command = ["delete_campus_aq", "-campus", campus]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_add_campus_building(self,
campus,
building,
fail=False,
errstr=""):
command = [
"add_campus_building_aq", "-campus_name", campus, "-building_name",
building
]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus_building(self,
campus,
building,
fail=False,
errstr=""):
command = [
"delete_campus_building_aq", "-campus_name", campus,
"-building_name", building
]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_verify(self, empty=False):
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
fail_expected_name = os.path.join(dsdb_coverage_dir,
DSDB_EXPECT_FAILURE_FILE)
issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE)
expected = {}
for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]:
expected_name = os.path.join(dsdb_coverage_dir, filename)
try:
with open(expected_name, "r") as fp:
for line in fp:
expected[line.rstrip("\n")] = True
except IOError:
pass
# This is likely a logic error in the test
if not expected and not empty:
self.fail("dsdb_verify() called when no DSDB commands were "
"expected?!?")
issued = {}
try:
with open(issued_name, "r") as fp:
for line in fp:
issued[line.rstrip("\n")] = True
except IOError:
pass
errors = []
for cmd, dummy in expected.items():
if cmd not in issued:
errors.append("'%s'" % cmd)
# Unexpected DSDB commands are caught by the fake_dsdb script
if errors:
self.fail("The following expected DSDB commands were not called:"
"\n@@@\n%s\n@@@\n" % "\n".join(errors))
def verify_buildfiles(self,
domain,
object,
want_exist=True,
command='manage'):
qdir = self.config.get('broker', 'quattordir')
domaindir = os.path.join(qdir, 'build', 'xml', domain)
xmlfile = os.path.join(domaindir, object + self.profile_suffix)
depfile = os.path.join(domaindir, object + '.dep')
builddir = self.config.get('broker', 'builddir')
profile = os.path.join(builddir, 'domains', domain, 'profiles',
object + self.template_extension)
for f in [xmlfile, depfile, profile]:
if want_exist:
self.failUnless(
os.path.exists(f),
"Expecting %s to exist before running %s." % (f, command))
else:
self.failIf(
os.path.exists(f),
"Not expecting %s to exist after running %s." %
(f, command))
def demote_current_user(self, role="nobody"):
principal = self.config.get('unittest', 'principal')
command = ["permission", "--role", role, "--principal", principal]
self.noouttest(command)
def promote_current_user(self):
srcdir = self.config.get("broker", "srcdir")
add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py")
env = os.environ.copy()
env['AQDCONF'] = self.config.baseconfig
p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env)
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0,
"Failed to restore admin privs '%s', '%s'." % (out, err))
if not os.path.exists(opts.config):
print >> sys.stderr, "configfile %s does not exist" % opts.config
sys.exit(1)
if os.environ.get("AQDCONF") and (os.path.realpath(opts.config)
!= os.path.realpath(os.environ["AQDCONF"])):
force_yes("""Will ignore AQDCONF variable value:
%s
and use
%s
instead.""" % (os.environ["AQDCONF"], opts.config))
config = Config(configfile=opts.config)
if not config.has_section("unittest"):
config.add_section("unittest")
if not config.has_option("unittest", "srcdir"):
config.set("unittest", "srcdir", SRCDIR)
if opts.coverage:
config.set("unittest", "coverage", "True")
if opts.profile:
config.set("unittest", "profile", "True")
hostname = config.get("unittest", "hostname")
if hostname.find(".") < 0:
print >> sys.stderr, """
Some regression tests depend on the config value for hostname to be
fully qualified. Please set the config value manually since the default
on this system (%s) is a short name.
""" % hostname
sys.exit(1)
def makeService(self, options):
# Start up coverage ASAP.
coverage_dir = options["coveragedir"]
if coverage_dir:
os.makedirs(coverage_dir, 0755)
if options["coveragerc"]:
coveragerc = options["coveragerc"]
else:
coveragerc = None
self.coverage = coverage.coverage(config_file=coveragerc)
self.coverage.erase()
self.coverage.start()
# Get the config object.
config = Config(configfile=options["config"])
# Helper for finishing off the coverage report.
def stop_coverage():
log.msg("Finishing coverage")
self.coverage.stop()
aquilon_srcdir = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon")
sourcefiles = []
for dirpath, dirnames, filenames in os.walk(aquilon_srcdir):
# FIXME: try to do this from the coverage config file
if dirpath.endswith("aquilon"):
dirnames.remove("client")
elif dirpath.endswith("aqdb"):
dirnames.remove("utils")
for filename in filenames:
if not filename.endswith('.py'):
continue
sourcefiles.append(os.path.join(dirpath, filename))
self.coverage.html_report(sourcefiles, directory=coverage_dir)
self.coverage.xml_report(sourcefiles,
outfile=os.path.join(coverage_dir, "aqd.xml"))
with open(os.path.join(coverage_dir, "aqd.coverage"), "w") as outfile:
self.coverage.report(sourcefiles, file=outfile)
# Make sure the coverage report gets generated.
if coverage_dir:
reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage)
# Set up the environment...
m = Modulecmd()
log_module_load(m, config.get("broker", "CheckNet_module"))
if config.has_option("database", "module"):
log_module_load(m, config.get("database", "module"))
sys.path.append(config.get("protocols", "directory"))
# Set this up before the aqdb libs get imported...
integrate_logging(config)
progname = os.path.split(sys.argv[0])[1]
if progname == 'aqd':
if config.get('broker', 'mode') != 'readwrite':
log.msg("Broker started with aqd symlink, "
"setting config mode to readwrite")
config.set('broker', 'mode', 'readwrite')
if progname == 'aqd_readonly':
if config.get('broker', 'mode') != 'readonly':
log.msg("Broker started with aqd_readonly symlink, "
"setting config mode to readonly")
config.set('broker', 'mode', 'readonly')
log.msg("Loading broker in mode %s" % config.get('broker', 'mode'))
# Dynamic import means that we can parse config options before
# importing aqdb. This is a hack until aqdb can be imported without
# firing up database connections.
resources = __import__("aquilon.worker.resources", globals(), locals(),
["RestServer"], -1)
RestServer = getattr(resources, "RestServer")
restServer = RestServer(config)
openSite = AnonSite(restServer)
# twisted is nicely changing the umask for us when the process is
# set to daemonize. This sets it back.
restServer.set_umask()
reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask)
reactor.addSystemEventTrigger('after', 'startup',
restServer.set_thread_pool_size)
sockdir = config.get("broker", "sockdir")
if not os.path.exists(sockdir):
os.makedirs(sockdir, 0700)
os.chmod(sockdir, 0700)
if options["usesock"]:
return strports.service("unix:%s/aqdsock" % sockdir, openSite)
openport = config.get("broker", "openport")
if config.has_option("broker", "bind_address"):
bind_address = config.get("broker", "bind_address").strip()
openaddr = "tcp:%s:interface=%s" % (openport, bind_address)
else: # pragma: no cover
bind_address = None
openaddr = "tcp:%s" % openport
# Return before firing up knc.
if options["noauth"]:
return strports.service(openaddr, openSite)
sockname = os.path.join(sockdir, "kncsock")
# This flag controls whether or not this process will start up
# and monitor knc. Except for noauth mode knc has to be running,
# but this process doesn't have to be the thing that starts it up.
if config.getboolean("broker", "run_knc") or \
config.getboolean("broker", "run_git_daemon"):
mon = GracefulProcessMonitor()
# FIXME: Should probably run krb5_keytab here as well.
# and/or verify that the keytab file exists.
if config.getboolean("broker", "run_knc"):
keytab = config.get("broker", "keytab")
knc_args = ["/usr/bin/env",
"KRB5_KTNAME=FILE:%s" % keytab,
config.get("kerberos", "knc"), "-lS", sockname]
if bind_address:
knc_args.append("-a")
knc_args.append(bind_address)
knc_args.append(config.get("broker", "kncport"))
mon.addProcess("knc", knc_args)
if config.getboolean("broker", "run_git_daemon"):
# The git daemon *must* be invoked using the form 'git-daemon'
# instead of invoking git with a 'daemon' argument. The latter
# will fork and exec git-daemon, resulting in a new pid that
# the process monitor won't know about!
gitpath = config.get("broker", "git_path")
gitdaemon = config.get("broker", "git_daemon")
ospath = os.environ.get("PATH", "")
args = ["/usr/bin/env", "PATH=%s:%s" % (gitpath, ospath),
gitdaemon, "--export-all", "--base-path=%s" %
config.get("broker", "git_daemon_basedir")]
if config.has_option("broker", "git_port"):
args.append("--port=%s" % config.get("broker", "git_port"))
if bind_address:
args.append("--listen=%s" % bind_address)
args.append(config.get("broker", "kingdir"))
mon.addProcess("git-daemon", args)
mon.startService()
reactor.addSystemEventTrigger('before', 'shutdown', mon.stopService)
# This socket is created by twisted and only accessed by knc as
# connections come in.
if os.path.exists(sockname):
try:
log.msg("Attempting to remove old socket '%s'" % sockname)
os.remove(sockname)
log.msg("Succeeded removing old socket.")
except OSError, e:
log.msg("Could not remove old socket '%s': %s" % (sockname, e))
xtn_id = Column(GUID(), ForeignKey(Xtn.xtn_id, name='xtn_dtl_xtn_fk'),
nullable=False)
name = Column(String(255), nullable=False)
value = Column(String(255), default='True', nullable=False)
__table_args__ = (PrimaryKeyConstraint(xtn_id, name, value,
name="xtn_detail_pk"),
Index('xtn_dtl_name_idx', name,
oracle_compress=True),
Index('xtn_dtl_value_idx', value, oracle_compress=True),
{'oracle_compress': 'OLTP'})
Xtn.args = relationship(XtnDetail, lazy="joined", order_by=[XtnDetail.name])
if config.has_option('database', 'audit_schema'): # pragma: no cover
schema = config.get('database', 'audit_schema')
Xtn.__table__.schema = schema
XtnEnd.__table__.schema = schema
XtnDetail.__table__.schema = schema
def start_xtn(session, xtn_id, username, command, is_readonly, details, ignore):
""" Wrapper to log the start of a transaction (or running command).
Takes a dictionary with the transaction parameters. The keys are
command, usename, readonly, and details. The details parameter
is itself a a dictionary of option names to option values provided
for the command.
The options_to_split is a list of any options that need to be
class TestBrokerCommand(unittest.TestCase):
def setUp(self):
self.config = Config()
self.net = DummyNetworks()
# Need to import protocol buffers after we have the config
# object all squared away and we can set the sys.path
# variable appropriately.
# It would be simpler just to change sys.path in runtests.py,
# but this allows for each test to be run individually (without
# the runtests.py wrapper).
protodir = self.config.get("protocols", "directory")
if protodir not in sys.path:
sys.path.append(protodir)
for m in ['aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2',
'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2',
'aqdparamdefinitions_pb2', 'aqdparameters_pb2']:
globals()[m] = __import__(m)
self.user = self.config.get("broker", "user")
self.sandboxdir = os.path.join(self.config.get("broker",
"templatesdir"),
self.user)
self.template_extension = self.config.get("panc", "template_extension")
# This method is cumbersome. Should probably develop something
# like unittest.conf.defaults.
if self.config.has_option("unittest", "scratchdir"):
self.scratchdir = self.config.get("unittest", "scratchdir")
if not os.path.exists(self.scratchdir):
os.makedirs(self.scratchdir)
if self.config.has_option("unittest", "aurora_with_node"):
self.aurora_with_node = self.config.get("unittest",
"aurora_with_node")
else:
self.aurora_with_node = "oyidb1622"
if self.config.has_option("unittest", "aurora_without_node"):
self.aurora_without_node = self.config.get("unittest",
"aurora_without_node")
else:
self.aurora_without_node = "pissp1"
self.gzip_profiles = self.config.getboolean("panc", "gzip_output")
self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml"
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
for name in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE,
DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR]:
path = os.path.join(dsdb_coverage_dir, name)
try:
os.remove(path)
except OSError:
pass
def tearDown(self):
pass
def template_name(self, *template, **args):
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "template_name() called without domain or sandbox")
return os.path.join(dir, *template) + self.template_extension
def plenary_name(self, *template):
dir = self.config.get("broker", "plenarydir")
return os.path.join(dir, *template) + self.template_extension
def find_template(self, *template, **args):
""" Figure out the extension of an existing template """
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "find_template() called without domain or sandbox")
base = os.path.join(dir, *template)
for extension in [".tpl", ".pan"]:
if os.path.exists(base + extension):
return base + extension
self.assert_(0, "template %s does not exist with any extension" % base)
def build_profile_name(self, *template, **args):
base = os.path.join(self.config.get("broker", "builddir"),
"domains", args.get("domain"),
"profiles", *template)
return base + self.template_extension
msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n')
def runcommand(self, command, auth=True, **kwargs):
aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py")
if auth:
port = self.config.get("broker", "kncport")
else:
port = self.config.get("broker", "openport")
if isinstance(command, list):
args = [str(cmd) for cmd in command]
else:
args = [command]
args.insert(0, sys.executable)
args.insert(1, aq)
if "--aqport" not in args:
args.append("--aqport")
args.append(port)
if auth:
args.append("--aqservice")
args.append(self.config.get("broker", "service"))
else:
args.append("--noauth")
if "env" in kwargs:
# Make sure that kerberos tickets are still present if the
# environment is being overridden...
env = {}
for (key, value) in kwargs["env"].items():
env[key] = value
for (key, value) in os.environ.items():
if key.find("KRB") == 0 and key not in env:
env[key] = value
if 'USER' not in env:
env['USER'] = os.environ.get('USER', '')
kwargs["env"] = env
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Strip any msversion dev warnings out of STDERR
err = self.msversion_dev_re.sub('', err)
# Lock messages are pretty common...
err = err.replace('Client status messages disabled, '
'retries exceeded.\n', '')
err = LOCK_RE.sub('', err)
return (p, out, err)
def successtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def statustest(self, command, **kwargs):
(out, err) = self.successtest(command, **kwargs)
self.assertEmptyOut(out, command)
return err
def failuretest(self, command, returncode, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, returncode,
"Non-%s return code %s for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n"
% (returncode, p.returncode, command, out, err))
return (out, err)
def assertEmptyStream(self, name, contents, command):
self.assertEqual(contents, "",
"%s for %s was not empty:\n@@@\n'%s'\n@@@\n"
% (name, command, contents))
def assertEmptyErr(self, contents, command):
self.assertEmptyStream("STDERR", contents, command)
def assertEmptyOut(self, contents, command):
self.assertEmptyStream("STDOUT", contents, command)
def commandtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEmptyErr(err, command)
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n"
% (command, out))
return out
def noouttest(self, command, **kwargs):
out = self.commandtest(command, **kwargs)
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n"
% (command, out))
def ignoreoutputtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return
# Right now, commands are not implemented consistently. When that is
# addressed, this unit test should be updated.
def notfoundtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
if p.returncode == 0:
self.assertEqual(err, "",
"STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
else:
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(err.find("Not Found") >= 0,
"STDERR for %s did not include Not Found:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
def badrequesttest(self, command, ignoreout=False, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.failUnless(err.find("Bad Request") >= 0,
"STDERR for %s did not include Bad Request:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
if not ignoreout and "--debug" not in command:
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs):
(p, out, err) = self.runcommand(command, auth=auth, **kwargs)
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(err.find("Unauthorized:") >= 0,
"STDERR for %s did not include Unauthorized:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
if msgcheck:
self.searchoutput(err, r"Unauthorized (anonymous )?access attempt",
command)
return err
def internalerrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 5,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 5, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.assertEqual(err.find("Internal Server Error"), 0,
"STDERR for %s did not start with "
"Internal Server Error:\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
def unimplementederrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 5,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 5, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.assertEqual(err.find("Not Implemented"), 0,
"STDERR for %s did not start with "
"Not Implemented:\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
# Test for conflicting or invalid aq client options.
def badoptiontest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 2,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 2, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def partialerrortest(self, command, **kwargs):
# Currently these two cases behave the same way - same exit code
# and behavior.
return self.badoptiontest(command, **kwargs)
def matchoutput(self, out, s, command):
self.assert_(out.find(s) >= 0,
"output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def matchclean(self, out, s, command):
self.assert_(out.find(s) < 0,
"output for %s includes '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def searchoutput(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failUnless(m,
"output for %s did not match '%s':\n@@@\n'%s'\n@@@\n"
% (command, r, out))
return m
def searchclean(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failIf(m,
"output for %s matches '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
def parse_proto_msg(self, listclass, attr, msg, expect=None):
protolist = listclass()
protolist.ParseFromString(msg)
received = len(getattr(protolist, attr))
if expect is None:
self.failUnless(received > 0,
"No %s listed in %s protobuf message\n" %
(attr, listclass))
else:
self.failUnlessEqual(received, expect,
"%d %s expected, got %d\n" %
(expect, attr, received))
return protolist
def parse_netlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdnetworks_pb2.NetworkList,
'networks',
msg, expect)
def parse_hostlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.HostList,
'hosts',
msg, expect)
def parse_clusters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ClusterList,
'clusters',
msg, expect)
def parse_location_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdlocations_pb2.LocationList,
'locations',
msg, expect)
def parse_dns_domainlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList,
'dns_domains',
msg, expect)
def parse_service_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceList,
'services',
msg, expect)
def parse_servicemap_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceMapList,
'servicemaps',
msg, expect)
def parse_personality_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.PersonalityList,
'personalities',
msg, expect)
def parse_os_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList,
'operating_systems',
msg, expect)
def parse_audit_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdaudit_pb2.TransactionList,
'transactions', msg, expect)
def parse_resourcelist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ResourceList,
'resources',
msg, expect)
def parse_paramdefinition_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparamdefinitions_pb2.ParamDefinitionList,
'param_definitions', msg, expect)
def parse_parameters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparameters_pb2.ParameterList,
'parameters', msg, expect)
def gitenv(self, env=None):
"""Configure a known sanitised environment"""
git_path = self.config.get("broker", "git_path")
# The "publish" test abuses gitenv(), and it needs the Python interpreter
# in the path, because it runs the template unit tests which in turn
# call the aq command
python_path = os.path.dirname(sys.executable)
newenv = {}
newenv["USER"] = os.environ.get('USER', '')
if env:
for (key, value) in env.iteritems():
newenv[key] = value
if "PATH" in newenv:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, newenv["PATH"])
else:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, '/bin:/usr/bin')
return newenv
def gitcommand_raw(self, command, **kwargs):
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, "git")
env = self.gitenv(kwargs.pop("env", None))
p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
return p
def gitcommand(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def gitcommand_expectfailure(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.failIfEqual(p.returncode, 0,
"Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def check_git_merge_health(self, repo):
command = "merge HEAD"
out = self.gitcommand(command.split(" "), cwd=repo)
return
def grepcommand(self, command, **kwargs):
if self.config.has_option("unittest", "grep"):
grep = self.config.get("unittest", "grep")
else:
grep = "/bin/grep"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, grep)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
if p.returncode == 1:
return []
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
def findcommand(self, command, **kwargs):
if self.config.has_option("unittest", "find"):
find = self.config.get("unittest", "find")
else:
find = "/usr/bin/find"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, find)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
def writescratch(self, filename, contents):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'w') as f:
f.write(contents)
return scratchfile
def readscratch(self, filename):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'r') as f:
contents = f.read()
return contents
def dsdb_expect(self, command, fail=False, errstr=""):
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
if fail:
filename = DSDB_EXPECT_FAILURE_FILE
else:
filename = DSDB_EXPECT_SUCCESS_FILE
expected_name = os.path.join(dsdb_coverage_dir, filename)
with open(expected_name, "a") as fp:
if isinstance(command, list):
fp.write(" ".join([str(cmd) for cmd in command]))
else:
fp.write(str(command))
fp.write("\n")
if fail and errstr:
errfile = DSDB_EXPECT_FAILURE_ERROR
expected_name = os.path.join(dsdb_coverage_dir, errfile)
with open(expected_name, "a") as fp:
fp.write(errstr)
fp.write("\n")
def dsdb_expect_add(self, hostname, ip, interface=None, mac=None,
primary=None, comments=None, fail=False):
command = ["add_host", "-host_name", hostname,
"-ip_address", str(ip), "-status", "aq"]
if interface:
command.extend(["-interface_name",
str(interface).replace('/', '_')])
if mac:
command.extend(["-ethernet_address", str(mac)])
if primary:
command.extend(["-primary_host_name", primary])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_delete(self, ip, fail=False):
self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail)
def dsdb_expect_update(self, fqdn, iface=None, ip=None, mac=None,
comments=None, fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if iface:
command.extend(["-interface_name", iface])
if ip:
command.extend(["-ip_address", str(ip)])
if mac:
command.extend(["-ethernet_address", str(mac)])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_rename(self, fqdn, new_fqdn=None, iface=None,
new_iface=None, fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if new_fqdn:
command.extend(["-primary_host_name", new_fqdn])
if iface:
command.extend(["-interface_name", iface])
if new_iface:
command.extend(["-new_interface_name", new_iface])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_add_campus(self, campus, comments=None, fail=False,
errstr=""):
command = ["add_campus_aq", "-campus_name", campus]
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus(self, campus, fail=False, errstr=""):
command = ["delete_campus_aq", "-campus", campus]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_add_campus_building(self, campus, building, fail=False,
errstr=""):
command = ["add_campus_building_aq", "-campus_name", campus,
"-building_name", building]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus_building(self, campus, building, fail=False,
errstr=""):
command = ["delete_campus_building_aq", "-campus_name", campus,
"-building_name", building]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_verify(self, empty=False):
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
fail_expected_name = os.path.join(dsdb_coverage_dir,
DSDB_EXPECT_FAILURE_FILE)
issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE)
expected = {}
for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]:
expected_name = os.path.join(dsdb_coverage_dir, filename)
try:
with open(expected_name, "r") as fp:
for line in fp:
expected[line.rstrip("\n")] = True
except IOError:
pass
# This is likely a logic error in the test
if not expected and not empty:
self.fail("dsdb_verify() called when no DSDB commands were "
"expected?!?")
issued = {}
try:
with open(issued_name, "r") as fp:
for line in fp:
issued[line.rstrip("\n")] = True
except IOError:
pass
errors = []
for cmd, dummy in expected.items():
if cmd not in issued:
errors.append("'%s'" % cmd)
# Unexpected DSDB commands are caught by the fake_dsdb script
if errors:
self.fail("The following expected DSDB commands were not called:"
"\n@@@\n%s\n@@@\n" % "\n".join(errors))
def verify_buildfiles(self, domain, object,
want_exist=True, command='manage'):
qdir = self.config.get('broker', 'quattordir')
domaindir = os.path.join(qdir, 'build', 'xml', domain)
xmlfile = os.path.join(domaindir, object + self.profile_suffix)
depfile = os.path.join(domaindir, object + '.dep')
builddir = self.config.get('broker', 'builddir')
profile = os.path.join(builddir, 'domains', domain, 'profiles',
object + self.template_extension)
for f in [xmlfile, depfile, profile]:
if want_exist:
self.failUnless(os.path.exists(f),
"Expecting %s to exist before running %s." %
(f, command))
else:
self.failIf(os.path.exists(f),
"Not expecting %s to exist after running %s." %
(f, command))
def demote_current_user(self, role="nobody"):
principal = self.config.get('unittest', 'principal')
command = ["permission", "--role", role, "--principal", principal]
self.noouttest(command)
def promote_current_user(self):
srcdir = self.config.get("broker", "srcdir")
add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py")
env = os.environ.copy()
env['AQDCONF'] = self.config.baseconfig
p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env)
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Failed to restore admin privs '%s', '%s'." %
(out, err))
from sqlalchemy.orm import scoped_session, sessionmaker
from sqlalchemy.exc import DBAPIError, DatabaseError as SaDBError
from sqlalchemy.schema import CreateIndex
from sqlalchemy.dialects.oracle.base import OracleDDLCompiler
import ms.modulecmd as modcmd
try:
config = Config()
except Exception, e:
print >> sys.stderr, 'failed to read configuration: %s' % e
sys.exit(os.EX_CONFIG)
assert config, 'No configuration in db_factory'
if config.has_option("database", "module"):
modcmd.load(config.get("database", "module"))
# Add support for Oracle-specific index extensions
@compiles(CreateIndex, 'oracle')
def visit_create_index(create, compiler, **kw):
index = create.element
compiler._verify_index_table(index)
preparer = compiler.preparer
text = "CREATE "
if index.unique:
text += "UNIQUE "
if index.kwargs.get("oracle_bitmap", False):
text += "BITMAP "
value = Column(String(255), default='True', nullable=False)
__table_args__ = (PrimaryKeyConstraint(xtn_id,
name,
value,
name="xtn_dtl_pk"),
Index('xtn_dtl_name_idx', name, oracle_compress=True),
Index('xtn_dtl_value_idx', value,
oracle_compress=True), {
'oracle_compress': 'OLTP'
})
Xtn.args = relationship(XtnDetail, lazy="joined", order_by=[XtnDetail.name])
if config.has_option('database', 'audit_schema'): # pragma: no cover
schema = config.get('database', 'audit_schema')
Xtn.__table__.schema = schema
XtnEnd.__table__.schema = schema
XtnDetail.__table__.schema = schema
def start_xtn(session,
xtn_id,
username,
command,
is_readonly,
details,
options_to_split=None):
""" Wrapper to log the start of a transaction (or running command).
