def __init__(self, logger=LOGGER):
config = Config()
self.logger = logger
self.dsdb = config.get("broker", "dsdb")
self.dsdb_use_testdb = config.getboolean("broker", "dsdb_use_testdb")
self.location_sync = config.getboolean("broker", "dsdb_location_sync")
self.actions = []
self.rollback_list = []
def __init__(self, logger=LOGGER):
config = Config()
self.logger = logger
self.dsdb = config.get("broker", "dsdb")
self.dsdb_use_testdb = config.getboolean("broker", "dsdb_use_testdb")
self.location_sync = config.getboolean("broker", "dsdb_location_sync")
self.actions = []
self.rollback_list = []
def __init__(self, dbhost, logger=LOGGER):
super(PlenaryHost, self).__init__(logger=logger)
if not isinstance(dbhost, Host):
raise InternalError("PlenaryHost called with %s instead of Host" %
dbhost.__class__.name)
self.dbobj = dbhost
config = Config()
if config.getboolean("broker", "namespaced_host_profiles"):
self.plenaries.append(PlenaryNamespacedHost.get_plenary(dbhost))
if config.getboolean("broker", "flat_host_profiles"):
self.plenaries.append(PlenaryToplevelHost.get_plenary(dbhost))
self.plenaries.append(PlenaryHostData.get_plenary(dbhost))
def onEnter(self, dbcluster):
dbdecommissioned = HostLifecycle.get_unique(object_session(dbcluster),
"decommissioned",
compel=True)
config = Config()
archetype = dbcluster.personality.archetype
section = "archetype_" + archetype.name
opt = "allow_cascaded_deco"
if dbcluster.hosts and (not config.has_option(section, opt)
or not config.getboolean(section, opt)):
raise ArgumentError("Cannot change state to {0}, as {1}'s "
"archetype is {2}.".format(
dbdecommissioned.name, dbcluster,
archetype.name))
if dbcluster.machines:
raise ArgumentError("Cannot change state to {0}, as {1} has "
"{2} VM(s).".format(dbdecommissioned.name,
dbcluster,
len(dbcluster.machines)))
for dbhost in dbcluster.hosts:
dbhost.status.transition(dbhost, dbdecommissioned)
def onEnter(self, dbcluster):
dbdecommissioned = HostLifecycle.get_unique(object_session(dbcluster),
"decommissioned",
compel=True)
config = Config()
archetype = dbcluster.personality.archetype
section = "archetype_" + archetype.name
opt = "allow_cascaded_deco"
if dbcluster.hosts and (not config.has_option(section, opt) or
not config.getboolean(section, opt)):
raise ArgumentError("Cannot change state to {0}, as {1}'s "
"archetype is {2}."
.format(dbdecommissioned.name, dbcluster,
archetype.name))
if dbcluster.virtual_machines:
raise ArgumentError("Cannot change state to {0}, as {1} has "
"{2} VM(s)."
.format(dbdecommissioned.name, dbcluster,
len(dbcluster.virtual_machines)))
for dbhost in dbcluster.hosts:
dbhost.status.transition(dbhost, dbdecommissioned)
def teststart(self):
# FIXME: Either remove any old pidfiles, or ignore it as a warning
# from stderr... or IMHO (daqscott) if pid files exist and are knc or
# python processes, kill -9 the pids and delete the files (with a
# warning message it tickles you)
config = Config()
twistd = os.path.join(config.get("broker", "srcdir"), "lib",
"python2.6", "aquilon", "unittest_patches.py")
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
logfile = config.get("broker", "logfile")
# Specify twistd and options...
args = [
sys.executable, twistd, "--pidfile", pidfile, "--logfile", logfile
]
if config.has_option("unittest", "profile"):
if config.getboolean("unittest", "profile"):
args.append("--profile")
args.append(
os.path.join(config.get("broker", "logdir"),
"aqd.profile"))
args.append("--profiler=cProfile")
args.append("--savestats")
# And then aqd and options...
args.extend(["aqd", "--config", config.baseconfig])
if config.has_option("unittest", "coverage"):
if config.getboolean("unittest", "coverage"):
args.append("--coveragedir")
dir = os.path.join(config.get("broker", "logdir"), "coverage")
args.append(dir)
coveragerc = os.path.join(config.get("broker", "srcdir"),
"tests", "coverage.rc")
args.append("--coveragerc")
args.append(coveragerc)
p = Popen(args)
self.assertEqual(p.wait(), 0)
def teststart(self):
# FIXME: Either remove any old pidfiles, or ignore it as a warning
# from stderr... or IMHO (daqscott) if pid files exist and are knc or
# python processes, kill -9 the pids and delete the files (with a
# warning message it tickles you)
config = Config()
twistd = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon", "unittest_patches.py")
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
logfile = config.get("broker", "logfile")
# Specify twistd and options...
args = [sys.executable, twistd,
"--pidfile", pidfile, "--logfile", logfile]
if config.has_option("unittest", "profile"):
if config.getboolean("unittest", "profile"):
args.append("--profile")
args.append(os.path.join(config.get("broker", "logdir"),
"aqd.profile"))
args.append("--profiler=cProfile")
args.append("--savestats")
# And then aqd and options...
args.extend(["aqd", "--config", config.baseconfig])
if config.has_option("unittest", "coverage"):
if config.getboolean("unittest", "coverage"):
args.append("--coveragedir")
dir = os.path.join(config.get("broker", "logdir"), "coverage")
args.append(dir)
coveragerc = os.path.join(config.get("broker", "srcdir"),
"tests", "coverage.rc")
args.append("--coveragerc")
args.append(coveragerc)
p = Popen(args)
self.assertEqual(p.wait(), 0)
class PlenaryHost(PlenaryCollection):
"""
A facade for Toplevel and Namespaced Hosts (below).
This class creates either/both toplevel and namespaced host plenaries,
based on broker configuration:
namespaced_host_profiles (boolean):
if namespaced profiles should be generated
flat_host_profiles (boolean):
if host profiles should be put into a "flat" toplevel (non-namespaced)
"""
def __init__(self, dbhost, logger=LOGGER):
if not isinstance(dbhost, Host):
raise InternalError("PlenaryHost called with %s instead of Host" %
dbhost.__class__.name)
PlenaryCollection.__init__(self, logger=logger)
self.dbobj = dbhost
self.config = Config()
if self.config.getboolean("broker", "namespaced_host_profiles"):
self.plenaries.append(PlenaryNamespacedHost(dbhost))
if self.config.getboolean("broker", "flat_host_profiles"):
self.plenaries.append(PlenaryToplevelHost(dbhost))
self.plenaries.append(PlenaryHostData(dbhost))
def write(self, locked=False, content=None):
# Don't bother writing plenary files non-compilable archetypes.
if not self.dbobj.archetype.is_compileable:
return 0
# Standard PlenaryCollection swallows IncompleteError. If/when
# the Host plenaries no longer raise that error this override
# should be removed.
total = 0
for plenary in self.plenaries:
total += plenary.write(locked=locked, content=content)
return total
class PlenaryHost(PlenaryCollection):
"""
A facade for Toplevel and Namespaced Hosts (below).
This class creates either/both toplevel and namespaced host plenaries,
based on broker configuration:
namespaced_host_profiles (boolean):
if namespaced profiles should be generated
flat_host_profiles (boolean):
if host profiles should be put into a "flat" toplevel (non-namespaced)
"""
def __init__(self, dbhost, logger=LOGGER):
if not isinstance(dbhost, Host):
raise InternalError("PlenaryHost called with %s instead of Host" % dbhost.__class__.name)
PlenaryCollection.__init__(self, logger=logger)
self.dbobj = dbhost
self.config = Config()
if self.config.getboolean("broker", "namespaced_host_profiles"):
self.plenaries.append(PlenaryNamespacedHost(dbhost))
if self.config.getboolean("broker", "flat_host_profiles"):
self.plenaries.append(PlenaryToplevelHost(dbhost))
self.plenaries.append(PlenaryHostData(dbhost))
def write(self, locked=False, content=None):
# Don't bother writing plenary files non-compilable archetypes.
if not self.dbobj.archetype.is_compileable:
return 0
# Standard PlenaryCollection swallows IncompleteError. If/when
# the Host plenaries no longer raise that error this override
# should be removed.
total = 0
for plenary in self.plenaries:
total += plenary.write(locked=locked, content=content)
return total
class TestBrokerCommand(unittest.TestCase):
def setUp(self):
self.config = Config()
self.net = DummyNetworks()
# Need to import protocol buffers after we have the config
# object all squared away and we can set the sys.path
# variable appropriately.
# It would be simpler just to change sys.path in runtests.py,
# but this allows for each test to be run individually (without
# the runtests.py wrapper).
protodir = self.config.get("protocols", "directory")
if protodir not in sys.path:
sys.path.append(protodir)
for m in [
'aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2',
'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2',
'aqdparamdefinitions_pb2', 'aqdparameters_pb2'
]:
globals()[m] = __import__(m)
self.user = self.config.get("broker", "user")
self.sandboxdir = os.path.join(
self.config.get("broker", "templatesdir"), self.user)
self.template_extension = self.config.get("panc", "template_extension")
# This method is cumbersome. Should probably develop something
# like unittest.conf.defaults.
if self.config.has_option("unittest", "scratchdir"):
self.scratchdir = self.config.get("unittest", "scratchdir")
if not os.path.exists(self.scratchdir):
os.makedirs(self.scratchdir)
if self.config.has_option("unittest", "aurora_with_node"):
self.aurora_with_node = self.config.get("unittest",
"aurora_with_node")
else:
self.aurora_with_node = "oyidb1622"
if self.config.has_option("unittest", "aurora_without_node"):
self.aurora_without_node = self.config.get("unittest",
"aurora_without_node")
else:
self.aurora_without_node = "pissp1"
self.gzip_profiles = self.config.getboolean("panc", "gzip_output")
self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml"
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
for name in [
DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE,
DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR
]:
path = os.path.join(dsdb_coverage_dir, name)
try:
os.remove(path)
except OSError:
pass
def tearDown(self):
pass
def template_name(self, *template, **args):
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "template_name() called without domain or sandbox")
return os.path.join(dir, *template) + self.template_extension
def plenary_name(self, *template):
dir = self.config.get("broker", "plenarydir")
return os.path.join(dir, *template) + self.template_extension
def find_template(self, *template, **args):
""" Figure out the extension of an existing template """
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "find_template() called without domain or sandbox")
base = os.path.join(dir, *template)
for extension in [".tpl", ".pan"]:
if os.path.exists(base + extension):
return base + extension
self.assert_(0, "template %s does not exist with any extension" % base)
def build_profile_name(self, *template, **args):
base = os.path.join(self.config.get("broker", "builddir"), "domains",
args.get("domain"), "profiles", *template)
return base + self.template_extension
msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n')
def runcommand(self, command, auth=True, **kwargs):
aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py")
if auth:
port = self.config.get("broker", "kncport")
else:
port = self.config.get("broker", "openport")
if isinstance(command, list):
args = [str(cmd) for cmd in command]
else:
args = [command]
args.insert(0, sys.executable)
args.insert(1, aq)
if "--aqport" not in args:
args.append("--aqport")
args.append(port)
if auth:
args.append("--aqservice")
args.append(self.config.get("broker", "service"))
else:
args.append("--noauth")
if "env" in kwargs:
# Make sure that kerberos tickets are still present if the
# environment is being overridden...
env = {}
for (key, value) in kwargs["env"].items():
env[key] = value
for (key, value) in os.environ.items():
if key.find("KRB") == 0 and key not in env:
env[key] = value
if 'USER' not in env:
env['USER'] = os.environ.get('USER', '')
kwargs["env"] = env
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Strip any msversion dev warnings out of STDERR
err = self.msversion_dev_re.sub('', err)
# Lock messages are pretty common...
err = err.replace(
'Client status messages disabled, '
'retries exceeded.\n', '')
err = LOCK_RE.sub('', err)
return (p, out, err)
def successtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 0, "Non-zero return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err))
return (out, err)
def statustest(self, command, **kwargs):
(out, err) = self.successtest(command, **kwargs)
self.assertEmptyOut(out, command)
return err
def failuretest(self, command, returncode, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, returncode, "Non-%s return code %s for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n" %
(returncode, p.returncode, command, out, err))
return (out, err)
def assertEmptyStream(self, name, contents, command):
self.assertEqual(
contents, "", "%s for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(name, command, contents))
def assertEmptyErr(self, contents, command):
self.assertEmptyStream("STDERR", contents, command)
def assertEmptyOut(self, contents, command):
self.assertEmptyStream("STDOUT", contents, command)
def commandtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEmptyErr(err, command)
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n" %
(command, out))
return out
def noouttest(self, command, **kwargs):
out = self.commandtest(command, **kwargs)
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
def ignoreoutputtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return
# Right now, commands are not implemented consistently. When that is
# addressed, this unit test should be updated.
def notfoundtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
if p.returncode == 0:
self.assertEqual(
err, "", "STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, err))
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
else:
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(
err.find("Not Found") >= 0,
"STDERR for %s did not include Not Found:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
return err
def badrequesttest(self, command, ignoreout=False, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err))
self.failUnless(
err.find("Bad Request") >= 0,
"STDERR for %s did not include Bad Request:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
if not ignoreout and "--debug" not in command:
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs):
(p, out, err) = self.runcommand(command, auth=auth, **kwargs)
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.failUnless(
err.find("Unauthorized:") >= 0,
"STDERR for %s did not include Unauthorized:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
if msgcheck:
self.searchoutput(err, r"Unauthorized (anonymous )?access attempt",
command)
return err
def internalerrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 5, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.assertEqual(
err.find("Internal Server Error"), 0,
"STDERR for %s did not start with "
"Internal Server Error:\n@@@\n'%s'\n@@@\n" % (command, err))
return err
def unimplementederrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 5, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.assertEqual(
err.find("Not Implemented"), 0, "STDERR for %s did not start with "
"Not Implemented:\n@@@\n'%s'\n@@@\n" % (command, err))
return err
# Test for conflicting or invalid aq client options.
def badoptiontest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 2, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 2, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
return err
def partialerrortest(self, command, **kwargs):
# Currently these two cases behave the same way - same exit code
# and behavior.
return self.badoptiontest(command, **kwargs)
def matchoutput(self, out, s, command):
self.assert_(
out.find(s) >= 0,
"output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def matchclean(self, out, s, command):
self.assert_(
out.find(s) < 0, "output for %s includes '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def searchoutput(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failUnless(
m, "output for %s did not match '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
return m
def searchclean(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failIf(
m, "output for %s matches '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
def parse_proto_msg(self, listclass, attr, msg, expect=None):
protolist = listclass()
protolist.ParseFromString(msg)
received = len(getattr(protolist, attr))
if expect is None:
self.failUnless(
received > 0,
"No %s listed in %s protobuf message\n" % (attr, listclass))
else:
self.failUnlessEqual(
received, expect,
"%d %s expected, got %d\n" % (expect, attr, received))
return protolist
def parse_netlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdnetworks_pb2.NetworkList, 'networks',
msg, expect)
def parse_hostlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.HostList, 'hosts', msg,
expect)
def parse_clusters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ClusterList, 'clusters',
msg, expect)
def parse_location_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdlocations_pb2.LocationList, 'locations',
msg, expect)
def parse_dns_domainlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList,
'dns_domains', msg, expect)
def parse_service_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceList, 'services',
msg, expect)
def parse_servicemap_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceMapList,
'servicemaps', msg, expect)
def parse_personality_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.PersonalityList,
'personalities', msg, expect)
def parse_os_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList,
'operating_systems', msg, expect)
def parse_audit_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdaudit_pb2.TransactionList,
'transactions', msg, expect)
def parse_resourcelist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ResourceList, 'resources',
msg, expect)
def parse_paramdefinition_msg(self, msg, expect=None):
return self.parse_proto_msg(
aqdparamdefinitions_pb2.ParamDefinitionList, 'param_definitions',
msg, expect)
def parse_parameters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparameters_pb2.ParameterList,
'parameters', msg, expect)
def gitenv(self, env=None):
"""Configure a known sanitised environment"""
git_path = self.config.get("broker", "git_path")
# The "publish" test abuses gitenv(), and it needs the Python interpreter
# in the path, because it runs the template unit tests which in turn
# call the aq command
python_path = os.path.dirname(sys.executable)
newenv = {}
newenv["USER"] = os.environ.get('USER', '')
if env:
for (key, value) in env.iteritems():
newenv[key] = value
if "PATH" in newenv:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path,
newenv["PATH"])
else:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path,
'/bin:/usr/bin')
return newenv
def gitcommand_raw(self, command, **kwargs):
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, "git")
env = self.gitenv(kwargs.pop("env", None))
p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
return p
def gitcommand(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def gitcommand_expectfailure(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.failIfEqual(
p.returncode, 0,
"Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def check_git_merge_health(self, repo):
command = "merge HEAD"
out = self.gitcommand(command.split(" "), cwd=repo)
return
def grepcommand(self, command, **kwargs):
if self.config.has_option("unittest", "grep"):
grep = self.config.get("unittest", "grep")
else:
grep = "/bin/grep"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, grep)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
if p.returncode == 1:
return []
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(command, out, err))
def findcommand(self, command, **kwargs):
if self.config.has_option("unittest", "find"):
find = self.config.get("unittest", "find")
else:
find = "/usr/bin/find"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, find)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(command, out, err))
def writescratch(self, filename, contents):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'w') as f:
f.write(contents)
return scratchfile
def readscratch(self, filename):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'r') as f:
contents = f.read()
return contents
def dsdb_expect(self, command, fail=False, errstr=""):
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
if fail:
filename = DSDB_EXPECT_FAILURE_FILE
else:
filename = DSDB_EXPECT_SUCCESS_FILE
expected_name = os.path.join(dsdb_coverage_dir, filename)
with open(expected_name, "a") as fp:
if isinstance(command, list):
fp.write(" ".join([str(cmd) for cmd in command]))
else:
fp.write(str(command))
fp.write("\n")
if fail and errstr:
errfile = DSDB_EXPECT_FAILURE_ERROR
expected_name = os.path.join(dsdb_coverage_dir, errfile)
with open(expected_name, "a") as fp:
fp.write(errstr)
fp.write("\n")
def dsdb_expect_add(self,
hostname,
ip,
interface=None,
mac=None,
primary=None,
comments=None,
fail=False):
command = [
"add_host", "-host_name", hostname, "-ip_address",
str(ip), "-status", "aq"
]
if interface:
command.extend(
["-interface_name",
str(interface).replace('/', '_')])
if mac:
command.extend(["-ethernet_address", str(mac)])
if primary:
command.extend(["-primary_host_name", primary])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_delete(self, ip, fail=False):
self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail)
def dsdb_expect_update(self,
fqdn,
iface=None,
ip=None,
mac=None,
comments=None,
fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if iface:
command.extend(["-interface_name", iface])
if ip:
command.extend(["-ip_address", str(ip)])
if mac:
command.extend(["-ethernet_address", str(mac)])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_rename(self,
fqdn,
new_fqdn=None,
iface=None,
new_iface=None,
fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if new_fqdn:
command.extend(["-primary_host_name", new_fqdn])
if iface:
command.extend(["-interface_name", iface])
if new_iface:
command.extend(["-new_interface_name", new_iface])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_add_campus(self,
campus,
comments=None,
fail=False,
errstr=""):
command = ["add_campus_aq", "-campus_name", campus]
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus(self, campus, fail=False, errstr=""):
command = ["delete_campus_aq", "-campus", campus]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_add_campus_building(self,
campus,
building,
fail=False,
errstr=""):
command = [
"add_campus_building_aq", "-campus_name", campus, "-building_name",
building
]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus_building(self,
campus,
building,
fail=False,
errstr=""):
command = [
"delete_campus_building_aq", "-campus_name", campus,
"-building_name", building
]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_verify(self, empty=False):
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
fail_expected_name = os.path.join(dsdb_coverage_dir,
DSDB_EXPECT_FAILURE_FILE)
issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE)
expected = {}
for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]:
expected_name = os.path.join(dsdb_coverage_dir, filename)
try:
with open(expected_name, "r") as fp:
for line in fp:
expected[line.rstrip("\n")] = True
except IOError:
pass
# This is likely a logic error in the test
if not expected and not empty:
self.fail("dsdb_verify() called when no DSDB commands were "
"expected?!?")
issued = {}
try:
with open(issued_name, "r") as fp:
for line in fp:
issued[line.rstrip("\n")] = True
except IOError:
pass
errors = []
for cmd, dummy in expected.items():
if cmd not in issued:
errors.append("'%s'" % cmd)
# Unexpected DSDB commands are caught by the fake_dsdb script
if errors:
self.fail("The following expected DSDB commands were not called:"
"\n@@@\n%s\n@@@\n" % "\n".join(errors))
def verify_buildfiles(self,
domain,
object,
want_exist=True,
command='manage'):
qdir = self.config.get('broker', 'quattordir')
domaindir = os.path.join(qdir, 'build', 'xml', domain)
xmlfile = os.path.join(domaindir, object + self.profile_suffix)
depfile = os.path.join(domaindir, object + '.dep')
builddir = self.config.get('broker', 'builddir')
profile = os.path.join(builddir, 'domains', domain, 'profiles',
object + self.template_extension)
for f in [xmlfile, depfile, profile]:
if want_exist:
self.failUnless(
os.path.exists(f),
"Expecting %s to exist before running %s." % (f, command))
else:
self.failIf(
os.path.exists(f),
"Not expecting %s to exist after running %s." %
(f, command))
def demote_current_user(self, role="nobody"):
principal = self.config.get('unittest', 'principal')
command = ["permission", "--role", role, "--principal", principal]
self.noouttest(command)
def promote_current_user(self):
srcdir = self.config.get("broker", "srcdir")
add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py")
env = os.environ.copy()
env['AQDCONF'] = self.config.baseconfig
p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env)
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0,
"Failed to restore admin privs '%s', '%s'." % (out, err))
def makeService(self, options):
# Start up coverage ASAP.
coverage_dir = options["coveragedir"]
if coverage_dir:
os.makedirs(coverage_dir, 0755)
if options["coveragerc"]:
coveragerc = options["coveragerc"]
else:
coveragerc = None
self.coverage = coverage.coverage(config_file=coveragerc)
self.coverage.erase()
self.coverage.start()
# Get the config object.
config = Config(configfile=options["config"])
# Helper for finishing off the coverage report.
def stop_coverage():
log.msg("Finishing coverage")
self.coverage.stop()
aquilon_srcdir = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon")
sourcefiles = []
for dirpath, dirnames, filenames in os.walk(aquilon_srcdir):
# FIXME: try to do this from the coverage config file
if dirpath.endswith("aquilon"):
dirnames.remove("client")
elif dirpath.endswith("aqdb"):
dirnames.remove("utils")
for filename in filenames:
if not filename.endswith('.py'):
continue
sourcefiles.append(os.path.join(dirpath, filename))
self.coverage.html_report(sourcefiles, directory=coverage_dir)
self.coverage.xml_report(sourcefiles,
outfile=os.path.join(coverage_dir, "aqd.xml"))
with open(os.path.join(coverage_dir, "aqd.coverage"), "w") as outfile:
self.coverage.report(sourcefiles, file=outfile)
# Make sure the coverage report gets generated.
if coverage_dir:
reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage)
# Set up the environment...
m = Modulecmd()
log_module_load(m, config.get("broker", "CheckNet_module"))
if config.has_option("database", "module"):
log_module_load(m, config.get("database", "module"))
sys.path.append(config.get("protocols", "directory"))
# Set this up before the aqdb libs get imported...
integrate_logging(config)
progname = os.path.split(sys.argv[0])[1]
if progname == 'aqd':
if config.get('broker', 'mode') != 'readwrite':
log.msg("Broker started with aqd symlink, "
"setting config mode to readwrite")
config.set('broker', 'mode', 'readwrite')
if progname == 'aqd_readonly':
if config.get('broker', 'mode') != 'readonly':
log.msg("Broker started with aqd_readonly symlink, "
"setting config mode to readonly")
config.set('broker', 'mode', 'readonly')
log.msg("Loading broker in mode %s" % config.get('broker', 'mode'))
# Dynamic import means that we can parse config options before
# importing aqdb. This is a hack until aqdb can be imported without
# firing up database connections.
resources = __import__("aquilon.worker.resources", globals(), locals(),
["RestServer"], -1)
RestServer = getattr(resources, "RestServer")
restServer = RestServer(config)
openSite = AnonSite(restServer)
# twisted is nicely changing the umask for us when the process is
# set to daemonize. This sets it back.
restServer.set_umask()
reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask)
reactor.addSystemEventTrigger('after', 'startup',
restServer.set_thread_pool_size)
sockdir = config.get("broker", "sockdir")
if not os.path.exists(sockdir):
os.makedirs(sockdir, 0700)
os.chmod(sockdir, 0700)
if options["usesock"]:
return strports.service("unix:%s/aqdsock" % sockdir, openSite)
openport = config.get("broker", "openport")
if config.has_option("broker", "bind_address"):
bind_address = config.get("broker", "bind_address").strip()
openaddr = "tcp:%s:interface=%s" % (openport, bind_address)
else: # pragma: no cover
bind_address = None
openaddr = "tcp:%s" % openport
# Return before firing up knc.
if options["noauth"]:
return strports.service(openaddr, openSite)
sockname = os.path.join(sockdir, "kncsock")
# This flag controls whether or not this process will start up
# and monitor knc. Except for noauth mode knc has to be running,
# but this process doesn't have to be the thing that starts it up.
if config.getboolean("broker", "run_knc") or \
config.getboolean("broker", "run_git_daemon"):
mon = GracefulProcessMonitor()
# FIXME: Should probably run krb5_keytab here as well.
# and/or verify that the keytab file exists.
if config.getboolean("broker", "run_knc"):
keytab = config.get("broker", "keytab")
knc_args = ["/usr/bin/env",
"KRB5_KTNAME=FILE:%s" % keytab,
config.get("kerberos", "knc"), "-lS", sockname]
if bind_address:
knc_args.append("-a")
knc_args.append(bind_address)
knc_args.append(config.get("broker", "kncport"))
mon.addProcess("knc", knc_args)
if config.getboolean("broker", "run_git_daemon"):
# The git daemon *must* be invoked using the form 'git-daemon'
# instead of invoking git with a 'daemon' argument. The latter
# will fork and exec git-daemon, resulting in a new pid that
# the process monitor won't know about!
gitpath = config.get("broker", "git_path")
gitdaemon = config.get("broker", "git_daemon")
ospath = os.environ.get("PATH", "")
args = ["/usr/bin/env", "PATH=%s:%s" % (gitpath, ospath),
gitdaemon, "--export-all", "--base-path=%s" %
config.get("broker", "git_daemon_basedir")]
if config.has_option("broker", "git_port"):
args.append("--port=%s" % config.get("broker", "git_port"))
if bind_address:
args.append("--listen=%s" % bind_address)
args.append(config.get("broker", "kingdir"))
mon.addProcess("git-daemon", args)
mon.startService()
reactor.addSystemEventTrigger('before', 'shutdown', mon.stopService)
# This socket is created by twisted and only accessed by knc as
# connections come in.
if os.path.exists(sockname):
try:
log.msg("Attempting to remove old socket '%s'" % sockname)
os.remove(sockname)
log.msg("Succeeded removing old socket.")
except OSError, e:
log.msg("Could not remove old socket '%s': %s" % (sockname, e))
class TestBrokerCommand(unittest.TestCase):
def setUp(self):
self.config = Config()
self.net = DummyNetworks()
# Need to import protocol buffers after we have the config
# object all squared away and we can set the sys.path
# variable appropriately.
# It would be simpler just to change sys.path in runtests.py,
# but this allows for each test to be run individually (without
# the runtests.py wrapper).
protodir = self.config.get("protocols", "directory")
if protodir not in sys.path:
sys.path.append(protodir)
for m in ['aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2',
'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2',
'aqdparamdefinitions_pb2', 'aqdparameters_pb2']:
globals()[m] = __import__(m)
self.user = self.config.get("broker", "user")
self.sandboxdir = os.path.join(self.config.get("broker",
"templatesdir"),
self.user)
self.template_extension = self.config.get("panc", "template_extension")
# This method is cumbersome. Should probably develop something
# like unittest.conf.defaults.
if self.config.has_option("unittest", "scratchdir"):
self.scratchdir = self.config.get("unittest", "scratchdir")
if not os.path.exists(self.scratchdir):
os.makedirs(self.scratchdir)
if self.config.has_option("unittest", "aurora_with_node"):
self.aurora_with_node = self.config.get("unittest",
"aurora_with_node")
else:
self.aurora_with_node = "oyidb1622"
if self.config.has_option("unittest", "aurora_without_node"):
self.aurora_without_node = self.config.get("unittest",
"aurora_without_node")
else:
self.aurora_without_node = "pissp1"
self.gzip_profiles = self.config.getboolean("panc", "gzip_output")
self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml"
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
for name in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE,
DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR]:
path = os.path.join(dsdb_coverage_dir, name)
try:
os.remove(path)
except OSError:
pass
def tearDown(self):
pass
def template_name(self, *template, **args):
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "template_name() called without domain or sandbox")
return os.path.join(dir, *template) + self.template_extension
def plenary_name(self, *template):
dir = self.config.get("broker", "plenarydir")
return os.path.join(dir, *template) + self.template_extension
def find_template(self, *template, **args):
""" Figure out the extension of an existing template """
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "find_template() called without domain or sandbox")
base = os.path.join(dir, *template)
for extension in [".tpl", ".pan"]:
if os.path.exists(base + extension):
return base + extension
self.assert_(0, "template %s does not exist with any extension" % base)
def build_profile_name(self, *template, **args):
base = os.path.join(self.config.get("broker", "builddir"),
"domains", args.get("domain"),
"profiles", *template)
return base + self.template_extension
msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n')
def runcommand(self, command, auth=True, **kwargs):
aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py")
if auth:
port = self.config.get("broker", "kncport")
else:
port = self.config.get("broker", "openport")
if isinstance(command, list):
args = [str(cmd) for cmd in command]
else:
args = [command]
args.insert(0, sys.executable)
args.insert(1, aq)
if "--aqport" not in args:
args.append("--aqport")
args.append(port)
if auth:
args.append("--aqservice")
args.append(self.config.get("broker", "service"))
else:
args.append("--noauth")
if "env" in kwargs:
# Make sure that kerberos tickets are still present if the
# environment is being overridden...
env = {}
for (key, value) in kwargs["env"].items():
env[key] = value
for (key, value) in os.environ.items():
if key.find("KRB") == 0 and key not in env:
env[key] = value
if 'USER' not in env:
env['USER'] = os.environ.get('USER', '')
kwargs["env"] = env
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Strip any msversion dev warnings out of STDERR
err = self.msversion_dev_re.sub('', err)
# Lock messages are pretty common...
err = err.replace('Client status messages disabled, '
'retries exceeded.\n', '')
err = LOCK_RE.sub('', err)
return (p, out, err)
def successtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def statustest(self, command, **kwargs):
(out, err) = self.successtest(command, **kwargs)
self.assertEmptyOut(out, command)
return err
def failuretest(self, command, returncode, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, returncode,
"Non-%s return code %s for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n"
% (returncode, p.returncode, command, out, err))
return (out, err)
def assertEmptyStream(self, name, contents, command):
self.assertEqual(contents, "",
"%s for %s was not empty:\n@@@\n'%s'\n@@@\n"
% (name, command, contents))
def assertEmptyErr(self, contents, command):
self.assertEmptyStream("STDERR", contents, command)
def assertEmptyOut(self, contents, command):
self.assertEmptyStream("STDOUT", contents, command)
def commandtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEmptyErr(err, command)
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n"
% (command, out))
return out
def noouttest(self, command, **kwargs):
out = self.commandtest(command, **kwargs)
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n"
% (command, out))
def ignoreoutputtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return
# Right now, commands are not implemented consistently. When that is
# addressed, this unit test should be updated.
def notfoundtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
if p.returncode == 0:
self.assertEqual(err, "",
"STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
else:
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(err.find("Not Found") >= 0,
"STDERR for %s did not include Not Found:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
def badrequesttest(self, command, ignoreout=False, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.failUnless(err.find("Bad Request") >= 0,
"STDERR for %s did not include Bad Request:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
if not ignoreout and "--debug" not in command:
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs):
(p, out, err) = self.runcommand(command, auth=auth, **kwargs)
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(err.find("Unauthorized:") >= 0,
"STDERR for %s did not include Unauthorized:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
if msgcheck:
self.searchoutput(err, r"Unauthorized (anonymous )?access attempt",
command)
return err
def internalerrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 5,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 5, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.assertEqual(err.find("Internal Server Error"), 0,
"STDERR for %s did not start with "
"Internal Server Error:\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
def unimplementederrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 5,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 5, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.assertEqual(err.find("Not Implemented"), 0,
"STDERR for %s did not start with "
"Not Implemented:\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
# Test for conflicting or invalid aq client options.
def badoptiontest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 2,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 2, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def partialerrortest(self, command, **kwargs):
# Currently these two cases behave the same way - same exit code
# and behavior.
return self.badoptiontest(command, **kwargs)
def matchoutput(self, out, s, command):
self.assert_(out.find(s) >= 0,
"output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def matchclean(self, out, s, command):
self.assert_(out.find(s) < 0,
"output for %s includes '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def searchoutput(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failUnless(m,
"output for %s did not match '%s':\n@@@\n'%s'\n@@@\n"
% (command, r, out))
return m
def searchclean(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failIf(m,
"output for %s matches '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
def parse_proto_msg(self, listclass, attr, msg, expect=None):
protolist = listclass()
protolist.ParseFromString(msg)
received = len(getattr(protolist, attr))
if expect is None:
self.failUnless(received > 0,
"No %s listed in %s protobuf message\n" %
(attr, listclass))
else:
self.failUnlessEqual(received, expect,
"%d %s expected, got %d\n" %
(expect, attr, received))
return protolist
def parse_netlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdnetworks_pb2.NetworkList,
'networks',
msg, expect)
def parse_hostlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.HostList,
'hosts',
msg, expect)
def parse_clusters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ClusterList,
'clusters',
msg, expect)
def parse_location_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdlocations_pb2.LocationList,
'locations',
msg, expect)
def parse_dns_domainlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList,
'dns_domains',
msg, expect)
def parse_service_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceList,
'services',
msg, expect)
def parse_servicemap_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceMapList,
'servicemaps',
msg, expect)
def parse_personality_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.PersonalityList,
'personalities',
msg, expect)
def parse_os_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList,
'operating_systems',
msg, expect)
def parse_audit_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdaudit_pb2.TransactionList,
'transactions', msg, expect)
def parse_resourcelist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ResourceList,
'resources',
msg, expect)
def parse_paramdefinition_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparamdefinitions_pb2.ParamDefinitionList,
'param_definitions', msg, expect)
def parse_parameters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparameters_pb2.ParameterList,
'parameters', msg, expect)
def gitenv(self, env=None):
"""Configure a known sanitised environment"""
git_path = self.config.get("broker", "git_path")
# The "publish" test abuses gitenv(), and it needs the Python interpreter
# in the path, because it runs the template unit tests which in turn
# call the aq command
python_path = os.path.dirname(sys.executable)
newenv = {}
newenv["USER"] = os.environ.get('USER', '')
if env:
for (key, value) in env.iteritems():
newenv[key] = value
if "PATH" in newenv:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, newenv["PATH"])
else:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, '/bin:/usr/bin')
return newenv
def gitcommand_raw(self, command, **kwargs):
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, "git")
env = self.gitenv(kwargs.pop("env", None))
p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
return p
def gitcommand(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def gitcommand_expectfailure(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.failIfEqual(p.returncode, 0,
"Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def check_git_merge_health(self, repo):
command = "merge HEAD"
out = self.gitcommand(command.split(" "), cwd=repo)
return
def grepcommand(self, command, **kwargs):
if self.config.has_option("unittest", "grep"):
grep = self.config.get("unittest", "grep")
else:
grep = "/bin/grep"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, grep)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
if p.returncode == 1:
return []
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
def findcommand(self, command, **kwargs):
if self.config.has_option("unittest", "find"):
find = self.config.get("unittest", "find")
else:
find = "/usr/bin/find"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, find)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
def writescratch(self, filename, contents):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'w') as f:
f.write(contents)
return scratchfile
def readscratch(self, filename):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'r') as f:
contents = f.read()
return contents
def dsdb_expect(self, command, fail=False, errstr=""):
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
if fail:
filename = DSDB_EXPECT_FAILURE_FILE
else:
filename = DSDB_EXPECT_SUCCESS_FILE
expected_name = os.path.join(dsdb_coverage_dir, filename)
with open(expected_name, "a") as fp:
if isinstance(command, list):
fp.write(" ".join([str(cmd) for cmd in command]))
else:
fp.write(str(command))
fp.write("\n")
if fail and errstr:
errfile = DSDB_EXPECT_FAILURE_ERROR
expected_name = os.path.join(dsdb_coverage_dir, errfile)
with open(expected_name, "a") as fp:
fp.write(errstr)
fp.write("\n")
def dsdb_expect_add(self, hostname, ip, interface=None, mac=None,
primary=None, comments=None, fail=False):
command = ["add_host", "-host_name", hostname,
"-ip_address", str(ip), "-status", "aq"]
if interface:
command.extend(["-interface_name",
str(interface).replace('/', '_')])
if mac:
command.extend(["-ethernet_address", str(mac)])
if primary:
command.extend(["-primary_host_name", primary])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_delete(self, ip, fail=False):
self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail)
def dsdb_expect_update(self, fqdn, iface=None, ip=None, mac=None,
comments=None, fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if iface:
command.extend(["-interface_name", iface])
if ip:
command.extend(["-ip_address", str(ip)])
if mac:
command.extend(["-ethernet_address", str(mac)])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_rename(self, fqdn, new_fqdn=None, iface=None,
new_iface=None, fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if new_fqdn:
command.extend(["-primary_host_name", new_fqdn])
if iface:
command.extend(["-interface_name", iface])
if new_iface:
command.extend(["-new_interface_name", new_iface])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_add_campus(self, campus, comments=None, fail=False,
errstr=""):
command = ["add_campus_aq", "-campus_name", campus]
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus(self, campus, fail=False, errstr=""):
command = ["delete_campus_aq", "-campus", campus]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_add_campus_building(self, campus, building, fail=False,
errstr=""):
command = ["add_campus_building_aq", "-campus_name", campus,
"-building_name", building]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus_building(self, campus, building, fail=False,
errstr=""):
command = ["delete_campus_building_aq", "-campus_name", campus,
"-building_name", building]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_verify(self, empty=False):
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
fail_expected_name = os.path.join(dsdb_coverage_dir,
DSDB_EXPECT_FAILURE_FILE)
issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE)
expected = {}
for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]:
expected_name = os.path.join(dsdb_coverage_dir, filename)
try:
with open(expected_name, "r") as fp:
for line in fp:
expected[line.rstrip("\n")] = True
except IOError:
pass
# This is likely a logic error in the test
if not expected and not empty:
self.fail("dsdb_verify() called when no DSDB commands were "
"expected?!?")
issued = {}
try:
with open(issued_name, "r") as fp:
for line in fp:
issued[line.rstrip("\n")] = True
except IOError:
pass
errors = []
for cmd, dummy in expected.items():
if cmd not in issued:
errors.append("'%s'" % cmd)
# Unexpected DSDB commands are caught by the fake_dsdb script
if errors:
self.fail("The following expected DSDB commands were not called:"
"\n@@@\n%s\n@@@\n" % "\n".join(errors))
def verify_buildfiles(self, domain, object,
want_exist=True, command='manage'):
qdir = self.config.get('broker', 'quattordir')
domaindir = os.path.join(qdir, 'build', 'xml', domain)
xmlfile = os.path.join(domaindir, object + self.profile_suffix)
depfile = os.path.join(domaindir, object + '.dep')
builddir = self.config.get('broker', 'builddir')
profile = os.path.join(builddir, 'domains', domain, 'profiles',
object + self.template_extension)
for f in [xmlfile, depfile, profile]:
if want_exist:
self.failUnless(os.path.exists(f),
"Expecting %s to exist before running %s." %
(f, command))
else:
self.failIf(os.path.exists(f),
"Not expecting %s to exist after running %s." %
(f, command))
def demote_current_user(self, role="nobody"):
principal = self.config.get('unittest', 'principal')
command = ["permission", "--role", role, "--principal", principal]
self.noouttest(command)
def promote_current_user(self):
srcdir = self.config.get("broker", "srcdir")
add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py")
env = os.environ.copy()
env['AQDCONF'] = self.config.baseconfig
p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env)
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Failed to restore admin privs '%s', '%s'." %
(out, err))
def makeService(self, options):
# Start up coverage ASAP.
coverage_dir = options["coveragedir"]
if coverage_dir:
os.makedirs(coverage_dir, 0755)
if options["coveragerc"]:
coveragerc = options["coveragerc"]
else:
coveragerc = None
self.coverage = coverage.coverage(config_file=coveragerc)
self.coverage.erase()
self.coverage.start()
# Get the config object.
config = Config(configfile=options["config"])
# Helper for finishing off the coverage report.
def stop_coverage():
log.msg("Finishing coverage")
self.coverage.stop()
aquilon_srcdir = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon")
sourcefiles = []
for dirpath, dirnames, filenames in os.walk(aquilon_srcdir):
# FIXME: try to do this from the coverage config file
if dirpath.endswith("aquilon"):
dirnames.remove("client")
elif dirpath.endswith("aqdb"):
dirnames.remove("utils")
for filename in filenames:
if not filename.endswith('.py'):
continue
sourcefiles.append(os.path.join(dirpath, filename))
self.coverage.html_report(sourcefiles, directory=coverage_dir)
self.coverage.xml_report(sourcefiles,
outfile=os.path.join(
coverage_dir, "aqd.xml"))
with open(os.path.join(coverage_dir, "aqd.coverage"),
"w") as outfile:
self.coverage.report(sourcefiles, file=outfile)
# Make sure the coverage report gets generated.
if coverage_dir:
reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage)
# Set up the environment...
m = Modulecmd()
log_module_load(m, config.get("broker", "CheckNet_module"))
if config.has_option("database", "module"):
log_module_load(m, config.get("database", "module"))
sys.path.append(config.get("protocols", "directory"))
# Set this up before the aqdb libs get imported...
integrate_logging(config)
progname = os.path.split(sys.argv[0])[1]
if progname == 'aqd':
if config.get('broker', 'mode') != 'readwrite':
log.msg("Broker started with aqd symlink, "
"setting config mode to readwrite")
config.set('broker', 'mode', 'readwrite')
if progname == 'aqd_readonly':
if config.get('broker', 'mode') != 'readonly':
log.msg("Broker started with aqd_readonly symlink, "
"setting config mode to readonly")
config.set('broker', 'mode', 'readonly')
log.msg("Loading broker in mode %s" % config.get('broker', 'mode'))
# Dynamic import means that we can parse config options before
# importing aqdb. This is a hack until aqdb can be imported without
# firing up database connections.
resources = __import__("aquilon.worker.resources", globals(), locals(),
["RestServer"], -1)
RestServer = getattr(resources, "RestServer")
restServer = RestServer(config)
openSite = AnonSite(restServer)
# twisted is nicely changing the umask for us when the process is
# set to daemonize. This sets it back.
restServer.set_umask()
reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask)
reactor.addSystemEventTrigger('after', 'startup',
restServer.set_thread_pool_size)
sockdir = config.get("broker", "sockdir")
if not os.path.exists(sockdir):
os.makedirs(sockdir, 0700)
os.chmod(sockdir, 0700)
if options["usesock"]:
return strports.service("unix:%s/aqdsock" % sockdir, openSite)
openport = config.get("broker", "openport")
if config.has_option("broker", "bind_address"):
bind_address = config.get("broker", "bind_address").strip()
openaddr = "tcp:%s:interface=%s" % (openport, bind_address)
else: # pragma: no cover
bind_address = None
openaddr = "tcp:%s" % openport
# Return before firing up knc.
if options["noauth"]:
return strports.service(openaddr, openSite)
sockname = os.path.join(sockdir, "kncsock")
# This flag controls whether or not this process will start up
# and monitor knc. Except for noauth mode knc has to be running,
# but this process doesn't have to be the thing that starts it up.
if config.getboolean("broker", "run_knc") or \
config.getboolean("broker", "run_git_daemon"):
mon = GracefulProcessMonitor()
# FIXME: Should probably run krb5_keytab here as well.
# and/or verify that the keytab file exists.
if config.getboolean("broker", "run_knc"):
keytab = config.get("broker", "keytab")
knc_args = [
"/usr/bin/env",
"KRB5_KTNAME=FILE:%s" % keytab,
config.get("kerberos", "knc"), "-lS", sockname
]
if bind_address:
knc_args.append("-a")
knc_args.append(bind_address)
knc_args.append(config.get("broker", "kncport"))
mon.addProcess("knc", knc_args)
if config.getboolean("broker", "run_git_daemon"):
# The git daemon *must* be invoked using the form 'git-daemon'
# instead of invoking git with a 'daemon' argument. The latter
# will fork and exec git-daemon, resulting in a new pid that
# the process monitor won't know about!
gitpath = config.get("broker", "git_path")
gitdaemon = config.get("broker", "git_daemon")
ospath = os.environ.get("PATH", "")
args = [
"/usr/bin/env",
"PATH=%s:%s" % (gitpath, ospath), gitdaemon,
"--export-all",
"--base-path=%s" %
config.get("broker", "git_daemon_basedir")
]
if config.has_option("broker", "git_port"):
args.append("--port=%s" % config.get("broker", "git_port"))
if bind_address:
args.append("--listen=%s" % bind_address)
args.append(config.get("broker", "kingdir"))
mon.addProcess("git-daemon", args)
mon.startService()
reactor.addSystemEventTrigger('before', 'shutdown',
mon.stopService)
# This socket is created by twisted and only accessed by knc as
# connections come in.
if os.path.exists(sockname):
try:
log.msg("Attempting to remove old socket '%s'" % sockname)
os.remove(sockname)
log.msg("Succeeded removing old socket.")
except OSError, e:
log.msg("Could not remove old socket '%s': %s" % (sockname, e))
