def __init__(self, logger=LOGGER): config = Config() self.logger = logger self.dsdb = config.get("broker", "dsdb") self.dsdb_use_testdb = config.getboolean("broker", "dsdb_use_testdb") self.location_sync = config.getboolean("broker", "dsdb_location_sync") self.actions = [] self.rollback_list = []
def __init__(self, logger=LOGGER): config = Config() self.logger = logger self.dsdb = config.get("broker", "dsdb") self.dsdb_use_testdb = config.getboolean("broker", "dsdb_use_testdb") self.location_sync = config.getboolean("broker", "dsdb_location_sync") self.actions = [] self.rollback_list = []
def __init__(self, dbhost, logger=LOGGER): super(PlenaryHost, self).__init__(logger=logger) if not isinstance(dbhost, Host): raise InternalError("PlenaryHost called with %s instead of Host" % dbhost.__class__.name) self.dbobj = dbhost config = Config() if config.getboolean("broker", "namespaced_host_profiles"): self.plenaries.append(PlenaryNamespacedHost.get_plenary(dbhost)) if config.getboolean("broker", "flat_host_profiles"): self.plenaries.append(PlenaryToplevelHost.get_plenary(dbhost)) self.plenaries.append(PlenaryHostData.get_plenary(dbhost))
def onEnter(self, dbcluster): dbdecommissioned = HostLifecycle.get_unique(object_session(dbcluster), "decommissioned", compel=True) config = Config() archetype = dbcluster.personality.archetype section = "archetype_" + archetype.name opt = "allow_cascaded_deco" if dbcluster.hosts and (not config.has_option(section, opt) or not config.getboolean(section, opt)): raise ArgumentError("Cannot change state to {0}, as {1}'s " "archetype is {2}.".format( dbdecommissioned.name, dbcluster, archetype.name)) if dbcluster.machines: raise ArgumentError("Cannot change state to {0}, as {1} has " "{2} VM(s).".format(dbdecommissioned.name, dbcluster, len(dbcluster.machines))) for dbhost in dbcluster.hosts: dbhost.status.transition(dbhost, dbdecommissioned)
def onEnter(self, dbcluster): dbdecommissioned = HostLifecycle.get_unique(object_session(dbcluster), "decommissioned", compel=True) config = Config() archetype = dbcluster.personality.archetype section = "archetype_" + archetype.name opt = "allow_cascaded_deco" if dbcluster.hosts and (not config.has_option(section, opt) or not config.getboolean(section, opt)): raise ArgumentError("Cannot change state to {0}, as {1}'s " "archetype is {2}." .format(dbdecommissioned.name, dbcluster, archetype.name)) if dbcluster.virtual_machines: raise ArgumentError("Cannot change state to {0}, as {1} has " "{2} VM(s)." .format(dbdecommissioned.name, dbcluster, len(dbcluster.virtual_machines))) for dbhost in dbcluster.hosts: dbhost.status.transition(dbhost, dbdecommissioned)
def teststart(self): # FIXME: Either remove any old pidfiles, or ignore it as a warning # from stderr... or IMHO (daqscott) if pid files exist and are knc or # python processes, kill -9 the pids and delete the files (with a # warning message it tickles you) config = Config() twistd = os.path.join(config.get("broker", "srcdir"), "lib", "python2.6", "aquilon", "unittest_patches.py") pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid") logfile = config.get("broker", "logfile") # Specify twistd and options... args = [ sys.executable, twistd, "--pidfile", pidfile, "--logfile", logfile ] if config.has_option("unittest", "profile"): if config.getboolean("unittest", "profile"): args.append("--profile") args.append( os.path.join(config.get("broker", "logdir"), "aqd.profile")) args.append("--profiler=cProfile") args.append("--savestats") # And then aqd and options... args.extend(["aqd", "--config", config.baseconfig]) if config.has_option("unittest", "coverage"): if config.getboolean("unittest", "coverage"): args.append("--coveragedir") dir = os.path.join(config.get("broker", "logdir"), "coverage") args.append(dir) coveragerc = os.path.join(config.get("broker", "srcdir"), "tests", "coverage.rc") args.append("--coveragerc") args.append(coveragerc) p = Popen(args) self.assertEqual(p.wait(), 0)
def teststart(self): # FIXME: Either remove any old pidfiles, or ignore it as a warning # from stderr... or IMHO (daqscott) if pid files exist and are knc or # python processes, kill -9 the pids and delete the files (with a # warning message it tickles you) config = Config() twistd = os.path.join(config.get("broker", "srcdir"), "lib", "python2.6", "aquilon", "unittest_patches.py") pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid") logfile = config.get("broker", "logfile") # Specify twistd and options... args = [sys.executable, twistd, "--pidfile", pidfile, "--logfile", logfile] if config.has_option("unittest", "profile"): if config.getboolean("unittest", "profile"): args.append("--profile") args.append(os.path.join(config.get("broker", "logdir"), "aqd.profile")) args.append("--profiler=cProfile") args.append("--savestats") # And then aqd and options... args.extend(["aqd", "--config", config.baseconfig]) if config.has_option("unittest", "coverage"): if config.getboolean("unittest", "coverage"): args.append("--coveragedir") dir = os.path.join(config.get("broker", "logdir"), "coverage") args.append(dir) coveragerc = os.path.join(config.get("broker", "srcdir"), "tests", "coverage.rc") args.append("--coveragerc") args.append(coveragerc) p = Popen(args) self.assertEqual(p.wait(), 0)
class PlenaryHost(PlenaryCollection): """ A facade for Toplevel and Namespaced Hosts (below). This class creates either/both toplevel and namespaced host plenaries, based on broker configuration: namespaced_host_profiles (boolean): if namespaced profiles should be generated flat_host_profiles (boolean): if host profiles should be put into a "flat" toplevel (non-namespaced) """ def __init__(self, dbhost, logger=LOGGER): if not isinstance(dbhost, Host): raise InternalError("PlenaryHost called with %s instead of Host" % dbhost.__class__.name) PlenaryCollection.__init__(self, logger=logger) self.dbobj = dbhost self.config = Config() if self.config.getboolean("broker", "namespaced_host_profiles"): self.plenaries.append(PlenaryNamespacedHost(dbhost)) if self.config.getboolean("broker", "flat_host_profiles"): self.plenaries.append(PlenaryToplevelHost(dbhost)) self.plenaries.append(PlenaryHostData(dbhost)) def write(self, locked=False, content=None): # Don't bother writing plenary files non-compilable archetypes. if not self.dbobj.archetype.is_compileable: return 0 # Standard PlenaryCollection swallows IncompleteError. If/when # the Host plenaries no longer raise that error this override # should be removed. total = 0 for plenary in self.plenaries: total += plenary.write(locked=locked, content=content) return total
class PlenaryHost(PlenaryCollection): """ A facade for Toplevel and Namespaced Hosts (below). This class creates either/both toplevel and namespaced host plenaries, based on broker configuration: namespaced_host_profiles (boolean): if namespaced profiles should be generated flat_host_profiles (boolean): if host profiles should be put into a "flat" toplevel (non-namespaced) """ def __init__(self, dbhost, logger=LOGGER): if not isinstance(dbhost, Host): raise InternalError("PlenaryHost called with %s instead of Host" % dbhost.__class__.name) PlenaryCollection.__init__(self, logger=logger) self.dbobj = dbhost self.config = Config() if self.config.getboolean("broker", "namespaced_host_profiles"): self.plenaries.append(PlenaryNamespacedHost(dbhost)) if self.config.getboolean("broker", "flat_host_profiles"): self.plenaries.append(PlenaryToplevelHost(dbhost)) self.plenaries.append(PlenaryHostData(dbhost)) def write(self, locked=False, content=None): # Don't bother writing plenary files non-compilable archetypes. if not self.dbobj.archetype.is_compileable: return 0 # Standard PlenaryCollection swallows IncompleteError. If/when # the Host plenaries no longer raise that error this override # should be removed. total = 0 for plenary in self.plenaries: total += plenary.write(locked=locked, content=content) return total
class TestBrokerCommand(unittest.TestCase): def setUp(self): self.config = Config() self.net = DummyNetworks() # Need to import protocol buffers after we have the config # object all squared away and we can set the sys.path # variable appropriately. # It would be simpler just to change sys.path in runtests.py, # but this allows for each test to be run individually (without # the runtests.py wrapper). protodir = self.config.get("protocols", "directory") if protodir not in sys.path: sys.path.append(protodir) for m in [ 'aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2', 'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2', 'aqdparamdefinitions_pb2', 'aqdparameters_pb2' ]: globals()[m] = __import__(m) self.user = self.config.get("broker", "user") self.sandboxdir = os.path.join( self.config.get("broker", "templatesdir"), self.user) self.template_extension = self.config.get("panc", "template_extension") # This method is cumbersome. Should probably develop something # like unittest.conf.defaults. if self.config.has_option("unittest", "scratchdir"): self.scratchdir = self.config.get("unittest", "scratchdir") if not os.path.exists(self.scratchdir): os.makedirs(self.scratchdir) if self.config.has_option("unittest", "aurora_with_node"): self.aurora_with_node = self.config.get("unittest", "aurora_with_node") else: self.aurora_with_node = "oyidb1622" if self.config.has_option("unittest", "aurora_without_node"): self.aurora_without_node = self.config.get("unittest", "aurora_without_node") else: self.aurora_without_node = "pissp1" self.gzip_profiles = self.config.getboolean("panc", "gzip_output") self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml" dsdb_coverage_dir = os.path.join( self.config.get("unittest", "scratchdir"), "dsdb_coverage") for name in [ DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE, DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR ]: path = os.path.join(dsdb_coverage_dir, name) try: os.remove(path) except OSError: pass def tearDown(self): pass def template_name(self, *template, **args): if args.get("sandbox", None): dir = os.path.join(self.sandboxdir, args.get("sandbox")) elif args.get("domain", None): dir = os.path.join(self.config.get("broker", "domainsdir"), args.get("domain")) else: self.assert_(0, "template_name() called without domain or sandbox") return os.path.join(dir, *template) + self.template_extension def plenary_name(self, *template): dir = self.config.get("broker", "plenarydir") return os.path.join(dir, *template) + self.template_extension def find_template(self, *template, **args): """ Figure out the extension of an existing template """ if args.get("sandbox", None): dir = os.path.join(self.sandboxdir, args.get("sandbox")) elif args.get("domain", None): dir = os.path.join(self.config.get("broker", "domainsdir"), args.get("domain")) else: self.assert_(0, "find_template() called without domain or sandbox") base = os.path.join(dir, *template) for extension in [".tpl", ".pan"]: if os.path.exists(base + extension): return base + extension self.assert_(0, "template %s does not exist with any extension" % base) def build_profile_name(self, *template, **args): base = os.path.join(self.config.get("broker", "builddir"), "domains", args.get("domain"), "profiles", *template) return base + self.template_extension msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n') def runcommand(self, command, auth=True, **kwargs): aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py") if auth: port = self.config.get("broker", "kncport") else: port = self.config.get("broker", "openport") if isinstance(command, list): args = [str(cmd) for cmd in command] else: args = [command] args.insert(0, sys.executable) args.insert(1, aq) if "--aqport" not in args: args.append("--aqport") args.append(port) if auth: args.append("--aqservice") args.append(self.config.get("broker", "service")) else: args.append("--noauth") if "env" in kwargs: # Make sure that kerberos tickets are still present if the # environment is being overridden... env = {} for (key, value) in kwargs["env"].items(): env[key] = value for (key, value) in os.environ.items(): if key.find("KRB") == 0 and key not in env: env[key] = value if 'USER' not in env: env['USER'] = os.environ.get('USER', '') kwargs["env"] = env p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs) (out, err) = p.communicate() # Strip any msversion dev warnings out of STDERR err = self.msversion_dev_re.sub('', err) # Lock messages are pretty common... err = err.replace( 'Client status messages disabled, ' 'retries exceeded.\n', '') err = LOCK_RE.sub('', err) return (p, out, err) def successtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual( p.returncode, 0, "Non-zero return code for %s, " "STDOUT:\n@@@\n'%s'\n@@@\n" "STDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return (out, err) def statustest(self, command, **kwargs): (out, err) = self.successtest(command, **kwargs) self.assertEmptyOut(out, command) return err def failuretest(self, command, returncode, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual( p.returncode, returncode, "Non-%s return code %s for %s, " "STDOUT:\n@@@\n'%s'\n@@@\n" "STDERR:\n@@@\n'%s'\n@@@\n" % (returncode, p.returncode, command, out, err)) return (out, err) def assertEmptyStream(self, name, contents, command): self.assertEqual( contents, "", "%s for %s was not empty:\n@@@\n'%s'\n@@@\n" % (name, command, contents)) def assertEmptyErr(self, contents, command): self.assertEmptyStream("STDERR", contents, command) def assertEmptyOut(self, contents, command): self.assertEmptyStream("STDOUT", contents, command) def commandtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEmptyErr(err, command) self.assertEqual( p.returncode, 0, "Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n" % (command, out)) return out def noouttest(self, command, **kwargs): out = self.commandtest(command, **kwargs) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) def ignoreoutputtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) # Ignore out/err unless we get a non-zero return code, then log it. self.assertEqual( p.returncode, 0, "Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return # Right now, commands are not implemented consistently. When that is # addressed, this unit test should be updated. def notfoundtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) if p.returncode == 0: self.assertEqual( err, "", "STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, err)) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) else: self.assertEqual( p.returncode, 4, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err)) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.failUnless( err.find("Not Found") >= 0, "STDERR for %s did not include Not Found:" "\n@@@\n'%s'\n@@@\n" % (command, err)) return err def badrequesttest(self, command, ignoreout=False, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual( p.returncode, 4, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err)) self.failUnless( err.find("Bad Request") >= 0, "STDERR for %s did not include Bad Request:" "\n@@@\n'%s'\n@@@\n" % (command, err)) if not ignoreout and "--debug" not in command: self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) return err def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs): (p, out, err) = self.runcommand(command, auth=auth, **kwargs) self.assertEqual( p.returncode, 4, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err)) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.failUnless( err.find("Unauthorized:") >= 0, "STDERR for %s did not include Unauthorized:" "\n@@@\n'%s'\n@@@\n" % (command, err)) if msgcheck: self.searchoutput(err, r"Unauthorized (anonymous )?access attempt", command) return err def internalerrortest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual( p.returncode, 5, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err)) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.assertEqual( err.find("Internal Server Error"), 0, "STDERR for %s did not start with " "Internal Server Error:\n@@@\n'%s'\n@@@\n" % (command, err)) return err def unimplementederrortest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual( p.returncode, 5, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err)) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.assertEqual( err.find("Not Implemented"), 0, "STDERR for %s did not start with " "Not Implemented:\n@@@\n'%s'\n@@@\n" % (command, err)) return err # Test for conflicting or invalid aq client options. def badoptiontest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual( p.returncode, 2, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 2, out, err)) self.assertEqual( out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) return err def partialerrortest(self, command, **kwargs): # Currently these two cases behave the same way - same exit code # and behavior. return self.badoptiontest(command, **kwargs) def matchoutput(self, out, s, command): self.assert_( out.find(s) >= 0, "output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" % (command, s, out)) def matchclean(self, out, s, command): self.assert_( out.find(s) < 0, "output for %s includes '%s':\n@@@\n'%s'\n@@@\n" % (command, s, out)) def searchoutput(self, out, r, command): if isinstance(r, str): m = re.search(r, out, re.MULTILINE) else: m = re.search(r, out) self.failUnless( m, "output for %s did not match '%s':\n@@@\n'%s'\n@@@\n" % (command, r, out)) return m def searchclean(self, out, r, command): if isinstance(r, str): m = re.search(r, out, re.MULTILINE) else: m = re.search(r, out) self.failIf( m, "output for %s matches '%s':\n@@@\n'%s'\n@@@\n" % (command, r, out)) def parse_proto_msg(self, listclass, attr, msg, expect=None): protolist = listclass() protolist.ParseFromString(msg) received = len(getattr(protolist, attr)) if expect is None: self.failUnless( received > 0, "No %s listed in %s protobuf message\n" % (attr, listclass)) else: self.failUnlessEqual( received, expect, "%d %s expected, got %d\n" % (expect, attr, received)) return protolist def parse_netlist_msg(self, msg, expect=None): return self.parse_proto_msg(aqdnetworks_pb2.NetworkList, 'networks', msg, expect) def parse_hostlist_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.HostList, 'hosts', msg, expect) def parse_clusters_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.ClusterList, 'clusters', msg, expect) def parse_location_msg(self, msg, expect=None): return self.parse_proto_msg(aqdlocations_pb2.LocationList, 'locations', msg, expect) def parse_dns_domainlist_msg(self, msg, expect=None): return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList, 'dns_domains', msg, expect) def parse_service_msg(self, msg, expect=None): return self.parse_proto_msg(aqdservices_pb2.ServiceList, 'services', msg, expect) def parse_servicemap_msg(self, msg, expect=None): return self.parse_proto_msg(aqdservices_pb2.ServiceMapList, 'servicemaps', msg, expect) def parse_personality_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.PersonalityList, 'personalities', msg, expect) def parse_os_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList, 'operating_systems', msg, expect) def parse_audit_msg(self, msg, expect=None): return self.parse_proto_msg(aqdaudit_pb2.TransactionList, 'transactions', msg, expect) def parse_resourcelist_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.ResourceList, 'resources', msg, expect) def parse_paramdefinition_msg(self, msg, expect=None): return self.parse_proto_msg( aqdparamdefinitions_pb2.ParamDefinitionList, 'param_definitions', msg, expect) def parse_parameters_msg(self, msg, expect=None): return self.parse_proto_msg(aqdparameters_pb2.ParameterList, 'parameters', msg, expect) def gitenv(self, env=None): """Configure a known sanitised environment""" git_path = self.config.get("broker", "git_path") # The "publish" test abuses gitenv(), and it needs the Python interpreter # in the path, because it runs the template unit tests which in turn # call the aq command python_path = os.path.dirname(sys.executable) newenv = {} newenv["USER"] = os.environ.get('USER', '') if env: for (key, value) in env.iteritems(): newenv[key] = value if "PATH" in newenv: newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, newenv["PATH"]) else: newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, '/bin:/usr/bin') return newenv def gitcommand_raw(self, command, **kwargs): if isinstance(command, list): args = command[:] else: args = [command] args.insert(0, "git") env = self.gitenv(kwargs.pop("env", None)) p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs) return p def gitcommand(self, command, **kwargs): p = self.gitcommand_raw(command, **kwargs) # Ignore out/err unless we get a non-zero return code, then log it. (out, err) = p.communicate() self.assertEqual( p.returncode, 0, "Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return (out, err) def gitcommand_expectfailure(self, command, **kwargs): p = self.gitcommand_raw(command, **kwargs) # Ignore out/err unless we get a non-zero return code, then log it. (out, err) = p.communicate() self.failIfEqual( p.returncode, 0, "Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return (out, err) def check_git_merge_health(self, repo): command = "merge HEAD" out = self.gitcommand(command.split(" "), cwd=repo) return def grepcommand(self, command, **kwargs): if self.config.has_option("unittest", "grep"): grep = self.config.get("unittest", "grep") else: grep = "/bin/grep" if isinstance(command, list): args = command[:] else: args = [command] args.insert(0, grep) env = {} p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs) (out, err) = p.communicate() # Ignore out/err unless we get a non-zero return code, then log it. if p.returncode == 0: return out.splitlines() if p.returncode == 1: return [] self.fail("Error return code for %s, " "STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) def findcommand(self, command, **kwargs): if self.config.has_option("unittest", "find"): find = self.config.get("unittest", "find") else: find = "/usr/bin/find" if isinstance(command, list): args = command[:] else: args = [command] args.insert(0, find) env = {} p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs) (out, err) = p.communicate() # Ignore out/err unless we get a non-zero return code, then log it. if p.returncode == 0: return out.splitlines() self.fail("Error return code for %s, " "STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) def writescratch(self, filename, contents): scratchfile = os.path.join(self.scratchdir, filename) with open(scratchfile, 'w') as f: f.write(contents) return scratchfile def readscratch(self, filename): scratchfile = os.path.join(self.scratchdir, filename) with open(scratchfile, 'r') as f: contents = f.read() return contents def dsdb_expect(self, command, fail=False, errstr=""): dsdb_coverage_dir = os.path.join( self.config.get("unittest", "scratchdir"), "dsdb_coverage") if fail: filename = DSDB_EXPECT_FAILURE_FILE else: filename = DSDB_EXPECT_SUCCESS_FILE expected_name = os.path.join(dsdb_coverage_dir, filename) with open(expected_name, "a") as fp: if isinstance(command, list): fp.write(" ".join([str(cmd) for cmd in command])) else: fp.write(str(command)) fp.write("\n") if fail and errstr: errfile = DSDB_EXPECT_FAILURE_ERROR expected_name = os.path.join(dsdb_coverage_dir, errfile) with open(expected_name, "a") as fp: fp.write(errstr) fp.write("\n") def dsdb_expect_add(self, hostname, ip, interface=None, mac=None, primary=None, comments=None, fail=False): command = [ "add_host", "-host_name", hostname, "-ip_address", str(ip), "-status", "aq" ] if interface: command.extend( ["-interface_name", str(interface).replace('/', '_')]) if mac: command.extend(["-ethernet_address", str(mac)]) if primary: command.extend(["-primary_host_name", primary]) if comments: command.extend(["-comments", comments]) self.dsdb_expect(" ".join(command), fail=fail) def dsdb_expect_delete(self, ip, fail=False): self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail) def dsdb_expect_update(self, fqdn, iface=None, ip=None, mac=None, comments=None, fail=False): command = ["update_aqd_host", "-host_name", fqdn] if iface: command.extend(["-interface_name", iface]) if ip: command.extend(["-ip_address", str(ip)]) if mac: command.extend(["-ethernet_address", str(mac)]) if comments: command.extend(["-comments", comments]) self.dsdb_expect(" ".join(command), fail=fail) def dsdb_expect_rename(self, fqdn, new_fqdn=None, iface=None, new_iface=None, fail=False): command = ["update_aqd_host", "-host_name", fqdn] if new_fqdn: command.extend(["-primary_host_name", new_fqdn]) if iface: command.extend(["-interface_name", iface]) if new_iface: command.extend(["-new_interface_name", new_iface]) self.dsdb_expect(" ".join(command), fail=fail) def dsdb_expect_add_campus(self, campus, comments=None, fail=False, errstr=""): command = ["add_campus_aq", "-campus_name", campus] if comments: command.extend(["-comments", comments]) self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_expect_del_campus(self, campus, fail=False, errstr=""): command = ["delete_campus_aq", "-campus", campus] self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_expect_add_campus_building(self, campus, building, fail=False, errstr=""): command = [ "add_campus_building_aq", "-campus_name", campus, "-building_name", building ] self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_expect_del_campus_building(self, campus, building, fail=False, errstr=""): command = [ "delete_campus_building_aq", "-campus_name", campus, "-building_name", building ] self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_verify(self, empty=False): dsdb_coverage_dir = os.path.join( self.config.get("unittest", "scratchdir"), "dsdb_coverage") fail_expected_name = os.path.join(dsdb_coverage_dir, DSDB_EXPECT_FAILURE_FILE) issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE) expected = {} for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]: expected_name = os.path.join(dsdb_coverage_dir, filename) try: with open(expected_name, "r") as fp: for line in fp: expected[line.rstrip("\n")] = True except IOError: pass # This is likely a logic error in the test if not expected and not empty: self.fail("dsdb_verify() called when no DSDB commands were " "expected?!?") issued = {} try: with open(issued_name, "r") as fp: for line in fp: issued[line.rstrip("\n")] = True except IOError: pass errors = [] for cmd, dummy in expected.items(): if cmd not in issued: errors.append("'%s'" % cmd) # Unexpected DSDB commands are caught by the fake_dsdb script if errors: self.fail("The following expected DSDB commands were not called:" "\n@@@\n%s\n@@@\n" % "\n".join(errors)) def verify_buildfiles(self, domain, object, want_exist=True, command='manage'): qdir = self.config.get('broker', 'quattordir') domaindir = os.path.join(qdir, 'build', 'xml', domain) xmlfile = os.path.join(domaindir, object + self.profile_suffix) depfile = os.path.join(domaindir, object + '.dep') builddir = self.config.get('broker', 'builddir') profile = os.path.join(builddir, 'domains', domain, 'profiles', object + self.template_extension) for f in [xmlfile, depfile, profile]: if want_exist: self.failUnless( os.path.exists(f), "Expecting %s to exist before running %s." % (f, command)) else: self.failIf( os.path.exists(f), "Not expecting %s to exist after running %s." % (f, command)) def demote_current_user(self, role="nobody"): principal = self.config.get('unittest', 'principal') command = ["permission", "--role", role, "--principal", principal] self.noouttest(command) def promote_current_user(self): srcdir = self.config.get("broker", "srcdir") add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py") env = os.environ.copy() env['AQDCONF'] = self.config.baseconfig p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env) (out, err) = p.communicate() self.assertEqual( p.returncode, 0, "Failed to restore admin privs '%s', '%s'." % (out, err))
def makeService(self, options): # Start up coverage ASAP. coverage_dir = options["coveragedir"] if coverage_dir: os.makedirs(coverage_dir, 0755) if options["coveragerc"]: coveragerc = options["coveragerc"] else: coveragerc = None self.coverage = coverage.coverage(config_file=coveragerc) self.coverage.erase() self.coverage.start() # Get the config object. config = Config(configfile=options["config"]) # Helper for finishing off the coverage report. def stop_coverage(): log.msg("Finishing coverage") self.coverage.stop() aquilon_srcdir = os.path.join(config.get("broker", "srcdir"), "lib", "python2.6", "aquilon") sourcefiles = [] for dirpath, dirnames, filenames in os.walk(aquilon_srcdir): # FIXME: try to do this from the coverage config file if dirpath.endswith("aquilon"): dirnames.remove("client") elif dirpath.endswith("aqdb"): dirnames.remove("utils") for filename in filenames: if not filename.endswith('.py'): continue sourcefiles.append(os.path.join(dirpath, filename)) self.coverage.html_report(sourcefiles, directory=coverage_dir) self.coverage.xml_report(sourcefiles, outfile=os.path.join(coverage_dir, "aqd.xml")) with open(os.path.join(coverage_dir, "aqd.coverage"), "w") as outfile: self.coverage.report(sourcefiles, file=outfile) # Make sure the coverage report gets generated. if coverage_dir: reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage) # Set up the environment... m = Modulecmd() log_module_load(m, config.get("broker", "CheckNet_module")) if config.has_option("database", "module"): log_module_load(m, config.get("database", "module")) sys.path.append(config.get("protocols", "directory")) # Set this up before the aqdb libs get imported... integrate_logging(config) progname = os.path.split(sys.argv[0])[1] if progname == 'aqd': if config.get('broker', 'mode') != 'readwrite': log.msg("Broker started with aqd symlink, " "setting config mode to readwrite") config.set('broker', 'mode', 'readwrite') if progname == 'aqd_readonly': if config.get('broker', 'mode') != 'readonly': log.msg("Broker started with aqd_readonly symlink, " "setting config mode to readonly") config.set('broker', 'mode', 'readonly') log.msg("Loading broker in mode %s" % config.get('broker', 'mode')) # Dynamic import means that we can parse config options before # importing aqdb. This is a hack until aqdb can be imported without # firing up database connections. resources = __import__("aquilon.worker.resources", globals(), locals(), ["RestServer"], -1) RestServer = getattr(resources, "RestServer") restServer = RestServer(config) openSite = AnonSite(restServer) # twisted is nicely changing the umask for us when the process is # set to daemonize. This sets it back. restServer.set_umask() reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask) reactor.addSystemEventTrigger('after', 'startup', restServer.set_thread_pool_size) sockdir = config.get("broker", "sockdir") if not os.path.exists(sockdir): os.makedirs(sockdir, 0700) os.chmod(sockdir, 0700) if options["usesock"]: return strports.service("unix:%s/aqdsock" % sockdir, openSite) openport = config.get("broker", "openport") if config.has_option("broker", "bind_address"): bind_address = config.get("broker", "bind_address").strip() openaddr = "tcp:%s:interface=%s" % (openport, bind_address) else: # pragma: no cover bind_address = None openaddr = "tcp:%s" % openport # Return before firing up knc. if options["noauth"]: return strports.service(openaddr, openSite) sockname = os.path.join(sockdir, "kncsock") # This flag controls whether or not this process will start up # and monitor knc. Except for noauth mode knc has to be running, # but this process doesn't have to be the thing that starts it up. if config.getboolean("broker", "run_knc") or \ config.getboolean("broker", "run_git_daemon"): mon = GracefulProcessMonitor() # FIXME: Should probably run krb5_keytab here as well. # and/or verify that the keytab file exists. if config.getboolean("broker", "run_knc"): keytab = config.get("broker", "keytab") knc_args = ["/usr/bin/env", "KRB5_KTNAME=FILE:%s" % keytab, config.get("kerberos", "knc"), "-lS", sockname] if bind_address: knc_args.append("-a") knc_args.append(bind_address) knc_args.append(config.get("broker", "kncport")) mon.addProcess("knc", knc_args) if config.getboolean("broker", "run_git_daemon"): # The git daemon *must* be invoked using the form 'git-daemon' # instead of invoking git with a 'daemon' argument. The latter # will fork and exec git-daemon, resulting in a new pid that # the process monitor won't know about! gitpath = config.get("broker", "git_path") gitdaemon = config.get("broker", "git_daemon") ospath = os.environ.get("PATH", "") args = ["/usr/bin/env", "PATH=%s:%s" % (gitpath, ospath), gitdaemon, "--export-all", "--base-path=%s" % config.get("broker", "git_daemon_basedir")] if config.has_option("broker", "git_port"): args.append("--port=%s" % config.get("broker", "git_port")) if bind_address: args.append("--listen=%s" % bind_address) args.append(config.get("broker", "kingdir")) mon.addProcess("git-daemon", args) mon.startService() reactor.addSystemEventTrigger('before', 'shutdown', mon.stopService) # This socket is created by twisted and only accessed by knc as # connections come in. if os.path.exists(sockname): try: log.msg("Attempting to remove old socket '%s'" % sockname) os.remove(sockname) log.msg("Succeeded removing old socket.") except OSError, e: log.msg("Could not remove old socket '%s': %s" % (sockname, e))
class TestBrokerCommand(unittest.TestCase): def setUp(self): self.config = Config() self.net = DummyNetworks() # Need to import protocol buffers after we have the config # object all squared away and we can set the sys.path # variable appropriately. # It would be simpler just to change sys.path in runtests.py, # but this allows for each test to be run individually (without # the runtests.py wrapper). protodir = self.config.get("protocols", "directory") if protodir not in sys.path: sys.path.append(protodir) for m in ['aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2', 'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2', 'aqdparamdefinitions_pb2', 'aqdparameters_pb2']: globals()[m] = __import__(m) self.user = self.config.get("broker", "user") self.sandboxdir = os.path.join(self.config.get("broker", "templatesdir"), self.user) self.template_extension = self.config.get("panc", "template_extension") # This method is cumbersome. Should probably develop something # like unittest.conf.defaults. if self.config.has_option("unittest", "scratchdir"): self.scratchdir = self.config.get("unittest", "scratchdir") if not os.path.exists(self.scratchdir): os.makedirs(self.scratchdir) if self.config.has_option("unittest", "aurora_with_node"): self.aurora_with_node = self.config.get("unittest", "aurora_with_node") else: self.aurora_with_node = "oyidb1622" if self.config.has_option("unittest", "aurora_without_node"): self.aurora_without_node = self.config.get("unittest", "aurora_without_node") else: self.aurora_without_node = "pissp1" self.gzip_profiles = self.config.getboolean("panc", "gzip_output") self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml" dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"), "dsdb_coverage") for name in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE, DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR]: path = os.path.join(dsdb_coverage_dir, name) try: os.remove(path) except OSError: pass def tearDown(self): pass def template_name(self, *template, **args): if args.get("sandbox", None): dir = os.path.join(self.sandboxdir, args.get("sandbox")) elif args.get("domain", None): dir = os.path.join(self.config.get("broker", "domainsdir"), args.get("domain")) else: self.assert_(0, "template_name() called without domain or sandbox") return os.path.join(dir, *template) + self.template_extension def plenary_name(self, *template): dir = self.config.get("broker", "plenarydir") return os.path.join(dir, *template) + self.template_extension def find_template(self, *template, **args): """ Figure out the extension of an existing template """ if args.get("sandbox", None): dir = os.path.join(self.sandboxdir, args.get("sandbox")) elif args.get("domain", None): dir = os.path.join(self.config.get("broker", "domainsdir"), args.get("domain")) else: self.assert_(0, "find_template() called without domain or sandbox") base = os.path.join(dir, *template) for extension in [".tpl", ".pan"]: if os.path.exists(base + extension): return base + extension self.assert_(0, "template %s does not exist with any extension" % base) def build_profile_name(self, *template, **args): base = os.path.join(self.config.get("broker", "builddir"), "domains", args.get("domain"), "profiles", *template) return base + self.template_extension msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n') def runcommand(self, command, auth=True, **kwargs): aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py") if auth: port = self.config.get("broker", "kncport") else: port = self.config.get("broker", "openport") if isinstance(command, list): args = [str(cmd) for cmd in command] else: args = [command] args.insert(0, sys.executable) args.insert(1, aq) if "--aqport" not in args: args.append("--aqport") args.append(port) if auth: args.append("--aqservice") args.append(self.config.get("broker", "service")) else: args.append("--noauth") if "env" in kwargs: # Make sure that kerberos tickets are still present if the # environment is being overridden... env = {} for (key, value) in kwargs["env"].items(): env[key] = value for (key, value) in os.environ.items(): if key.find("KRB") == 0 and key not in env: env[key] = value if 'USER' not in env: env['USER'] = os.environ.get('USER', '') kwargs["env"] = env p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs) (out, err) = p.communicate() # Strip any msversion dev warnings out of STDERR err = self.msversion_dev_re.sub('', err) # Lock messages are pretty common... err = err.replace('Client status messages disabled, ' 'retries exceeded.\n', '') err = LOCK_RE.sub('', err) return (p, out, err) def successtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual(p.returncode, 0, "Non-zero return code for %s, " "STDOUT:\n@@@\n'%s'\n@@@\n" "STDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return (out, err) def statustest(self, command, **kwargs): (out, err) = self.successtest(command, **kwargs) self.assertEmptyOut(out, command) return err def failuretest(self, command, returncode, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual(p.returncode, returncode, "Non-%s return code %s for %s, " "STDOUT:\n@@@\n'%s'\n@@@\n" "STDERR:\n@@@\n'%s'\n@@@\n" % (returncode, p.returncode, command, out, err)) return (out, err) def assertEmptyStream(self, name, contents, command): self.assertEqual(contents, "", "%s for %s was not empty:\n@@@\n'%s'\n@@@\n" % (name, command, contents)) def assertEmptyErr(self, contents, command): self.assertEmptyStream("STDERR", contents, command) def assertEmptyOut(self, contents, command): self.assertEmptyStream("STDOUT", contents, command) def commandtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEmptyErr(err, command) self.assertEqual(p.returncode, 0, "Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n" % (command, out)) return out def noouttest(self, command, **kwargs): out = self.commandtest(command, **kwargs) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) def ignoreoutputtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) # Ignore out/err unless we get a non-zero return code, then log it. self.assertEqual(p.returncode, 0, "Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return # Right now, commands are not implemented consistently. When that is # addressed, this unit test should be updated. def notfoundtest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) if p.returncode == 0: self.assertEqual(err, "", "STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, err)) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) else: self.assertEqual(p.returncode, 4, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err)) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.failUnless(err.find("Not Found") >= 0, "STDERR for %s did not include Not Found:" "\n@@@\n'%s'\n@@@\n" % (command, err)) return err def badrequesttest(self, command, ignoreout=False, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual(p.returncode, 4, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err)) self.failUnless(err.find("Bad Request") >= 0, "STDERR for %s did not include Bad Request:" "\n@@@\n'%s'\n@@@\n" % (command, err)) if not ignoreout and "--debug" not in command: self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) return err def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs): (p, out, err) = self.runcommand(command, auth=auth, **kwargs) self.assertEqual(p.returncode, 4, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err)) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.failUnless(err.find("Unauthorized:") >= 0, "STDERR for %s did not include Unauthorized:" "\n@@@\n'%s'\n@@@\n" % (command, err)) if msgcheck: self.searchoutput(err, r"Unauthorized (anonymous )?access attempt", command) return err def internalerrortest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual(p.returncode, 5, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err)) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.assertEqual(err.find("Internal Server Error"), 0, "STDERR for %s did not start with " "Internal Server Error:\n@@@\n'%s'\n@@@\n" % (command, err)) return err def unimplementederrortest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual(p.returncode, 5, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err)) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) self.assertEqual(err.find("Not Implemented"), 0, "STDERR for %s did not start with " "Not Implemented:\n@@@\n'%s'\n@@@\n" % (command, err)) return err # Test for conflicting or invalid aq client options. def badoptiontest(self, command, **kwargs): (p, out, err) = self.runcommand(command, **kwargs) self.assertEqual(p.returncode, 2, "Return code for %s was %d instead of %d" "\nSTDOUT:\n@@@\n'%s'\n@@@" "\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 2, out, err)) self.assertEqual(out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out)) return err def partialerrortest(self, command, **kwargs): # Currently these two cases behave the same way - same exit code # and behavior. return self.badoptiontest(command, **kwargs) def matchoutput(self, out, s, command): self.assert_(out.find(s) >= 0, "output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" % (command, s, out)) def matchclean(self, out, s, command): self.assert_(out.find(s) < 0, "output for %s includes '%s':\n@@@\n'%s'\n@@@\n" % (command, s, out)) def searchoutput(self, out, r, command): if isinstance(r, str): m = re.search(r, out, re.MULTILINE) else: m = re.search(r, out) self.failUnless(m, "output for %s did not match '%s':\n@@@\n'%s'\n@@@\n" % (command, r, out)) return m def searchclean(self, out, r, command): if isinstance(r, str): m = re.search(r, out, re.MULTILINE) else: m = re.search(r, out) self.failIf(m, "output for %s matches '%s':\n@@@\n'%s'\n@@@\n" % (command, r, out)) def parse_proto_msg(self, listclass, attr, msg, expect=None): protolist = listclass() protolist.ParseFromString(msg) received = len(getattr(protolist, attr)) if expect is None: self.failUnless(received > 0, "No %s listed in %s protobuf message\n" % (attr, listclass)) else: self.failUnlessEqual(received, expect, "%d %s expected, got %d\n" % (expect, attr, received)) return protolist def parse_netlist_msg(self, msg, expect=None): return self.parse_proto_msg(aqdnetworks_pb2.NetworkList, 'networks', msg, expect) def parse_hostlist_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.HostList, 'hosts', msg, expect) def parse_clusters_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.ClusterList, 'clusters', msg, expect) def parse_location_msg(self, msg, expect=None): return self.parse_proto_msg(aqdlocations_pb2.LocationList, 'locations', msg, expect) def parse_dns_domainlist_msg(self, msg, expect=None): return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList, 'dns_domains', msg, expect) def parse_service_msg(self, msg, expect=None): return self.parse_proto_msg(aqdservices_pb2.ServiceList, 'services', msg, expect) def parse_servicemap_msg(self, msg, expect=None): return self.parse_proto_msg(aqdservices_pb2.ServiceMapList, 'servicemaps', msg, expect) def parse_personality_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.PersonalityList, 'personalities', msg, expect) def parse_os_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList, 'operating_systems', msg, expect) def parse_audit_msg(self, msg, expect=None): return self.parse_proto_msg(aqdaudit_pb2.TransactionList, 'transactions', msg, expect) def parse_resourcelist_msg(self, msg, expect=None): return self.parse_proto_msg(aqdsystems_pb2.ResourceList, 'resources', msg, expect) def parse_paramdefinition_msg(self, msg, expect=None): return self.parse_proto_msg(aqdparamdefinitions_pb2.ParamDefinitionList, 'param_definitions', msg, expect) def parse_parameters_msg(self, msg, expect=None): return self.parse_proto_msg(aqdparameters_pb2.ParameterList, 'parameters', msg, expect) def gitenv(self, env=None): """Configure a known sanitised environment""" git_path = self.config.get("broker", "git_path") # The "publish" test abuses gitenv(), and it needs the Python interpreter # in the path, because it runs the template unit tests which in turn # call the aq command python_path = os.path.dirname(sys.executable) newenv = {} newenv["USER"] = os.environ.get('USER', '') if env: for (key, value) in env.iteritems(): newenv[key] = value if "PATH" in newenv: newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, newenv["PATH"]) else: newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, '/bin:/usr/bin') return newenv def gitcommand_raw(self, command, **kwargs): if isinstance(command, list): args = command[:] else: args = [command] args.insert(0, "git") env = self.gitenv(kwargs.pop("env", None)) p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs) return p def gitcommand(self, command, **kwargs): p = self.gitcommand_raw(command, **kwargs) # Ignore out/err unless we get a non-zero return code, then log it. (out, err) = p.communicate() self.assertEqual(p.returncode, 0, "Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return (out, err) def gitcommand_expectfailure(self, command, **kwargs): p = self.gitcommand_raw(command, **kwargs) # Ignore out/err unless we get a non-zero return code, then log it. (out, err) = p.communicate() self.failIfEqual(p.returncode, 0, "Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) return (out, err) def check_git_merge_health(self, repo): command = "merge HEAD" out = self.gitcommand(command.split(" "), cwd=repo) return def grepcommand(self, command, **kwargs): if self.config.has_option("unittest", "grep"): grep = self.config.get("unittest", "grep") else: grep = "/bin/grep" if isinstance(command, list): args = command[:] else: args = [command] args.insert(0, grep) env = {} p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs) (out, err) = p.communicate() # Ignore out/err unless we get a non-zero return code, then log it. if p.returncode == 0: return out.splitlines() if p.returncode == 1: return [] self.fail("Error return code for %s, " "STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) def findcommand(self, command, **kwargs): if self.config.has_option("unittest", "find"): find = self.config.get("unittest", "find") else: find = "/usr/bin/find" if isinstance(command, list): args = command[:] else: args = [command] args.insert(0, find) env = {} p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs) (out, err) = p.communicate() # Ignore out/err unless we get a non-zero return code, then log it. if p.returncode == 0: return out.splitlines() self.fail("Error return code for %s, " "STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err)) def writescratch(self, filename, contents): scratchfile = os.path.join(self.scratchdir, filename) with open(scratchfile, 'w') as f: f.write(contents) return scratchfile def readscratch(self, filename): scratchfile = os.path.join(self.scratchdir, filename) with open(scratchfile, 'r') as f: contents = f.read() return contents def dsdb_expect(self, command, fail=False, errstr=""): dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"), "dsdb_coverage") if fail: filename = DSDB_EXPECT_FAILURE_FILE else: filename = DSDB_EXPECT_SUCCESS_FILE expected_name = os.path.join(dsdb_coverage_dir, filename) with open(expected_name, "a") as fp: if isinstance(command, list): fp.write(" ".join([str(cmd) for cmd in command])) else: fp.write(str(command)) fp.write("\n") if fail and errstr: errfile = DSDB_EXPECT_FAILURE_ERROR expected_name = os.path.join(dsdb_coverage_dir, errfile) with open(expected_name, "a") as fp: fp.write(errstr) fp.write("\n") def dsdb_expect_add(self, hostname, ip, interface=None, mac=None, primary=None, comments=None, fail=False): command = ["add_host", "-host_name", hostname, "-ip_address", str(ip), "-status", "aq"] if interface: command.extend(["-interface_name", str(interface).replace('/', '_')]) if mac: command.extend(["-ethernet_address", str(mac)]) if primary: command.extend(["-primary_host_name", primary]) if comments: command.extend(["-comments", comments]) self.dsdb_expect(" ".join(command), fail=fail) def dsdb_expect_delete(self, ip, fail=False): self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail) def dsdb_expect_update(self, fqdn, iface=None, ip=None, mac=None, comments=None, fail=False): command = ["update_aqd_host", "-host_name", fqdn] if iface: command.extend(["-interface_name", iface]) if ip: command.extend(["-ip_address", str(ip)]) if mac: command.extend(["-ethernet_address", str(mac)]) if comments: command.extend(["-comments", comments]) self.dsdb_expect(" ".join(command), fail=fail) def dsdb_expect_rename(self, fqdn, new_fqdn=None, iface=None, new_iface=None, fail=False): command = ["update_aqd_host", "-host_name", fqdn] if new_fqdn: command.extend(["-primary_host_name", new_fqdn]) if iface: command.extend(["-interface_name", iface]) if new_iface: command.extend(["-new_interface_name", new_iface]) self.dsdb_expect(" ".join(command), fail=fail) def dsdb_expect_add_campus(self, campus, comments=None, fail=False, errstr=""): command = ["add_campus_aq", "-campus_name", campus] if comments: command.extend(["-comments", comments]) self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_expect_del_campus(self, campus, fail=False, errstr=""): command = ["delete_campus_aq", "-campus", campus] self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_expect_add_campus_building(self, campus, building, fail=False, errstr=""): command = ["add_campus_building_aq", "-campus_name", campus, "-building_name", building] self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_expect_del_campus_building(self, campus, building, fail=False, errstr=""): command = ["delete_campus_building_aq", "-campus_name", campus, "-building_name", building] self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr) def dsdb_verify(self, empty=False): dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"), "dsdb_coverage") fail_expected_name = os.path.join(dsdb_coverage_dir, DSDB_EXPECT_FAILURE_FILE) issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE) expected = {} for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]: expected_name = os.path.join(dsdb_coverage_dir, filename) try: with open(expected_name, "r") as fp: for line in fp: expected[line.rstrip("\n")] = True except IOError: pass # This is likely a logic error in the test if not expected and not empty: self.fail("dsdb_verify() called when no DSDB commands were " "expected?!?") issued = {} try: with open(issued_name, "r") as fp: for line in fp: issued[line.rstrip("\n")] = True except IOError: pass errors = [] for cmd, dummy in expected.items(): if cmd not in issued: errors.append("'%s'" % cmd) # Unexpected DSDB commands are caught by the fake_dsdb script if errors: self.fail("The following expected DSDB commands were not called:" "\n@@@\n%s\n@@@\n" % "\n".join(errors)) def verify_buildfiles(self, domain, object, want_exist=True, command='manage'): qdir = self.config.get('broker', 'quattordir') domaindir = os.path.join(qdir, 'build', 'xml', domain) xmlfile = os.path.join(domaindir, object + self.profile_suffix) depfile = os.path.join(domaindir, object + '.dep') builddir = self.config.get('broker', 'builddir') profile = os.path.join(builddir, 'domains', domain, 'profiles', object + self.template_extension) for f in [xmlfile, depfile, profile]: if want_exist: self.failUnless(os.path.exists(f), "Expecting %s to exist before running %s." % (f, command)) else: self.failIf(os.path.exists(f), "Not expecting %s to exist after running %s." % (f, command)) def demote_current_user(self, role="nobody"): principal = self.config.get('unittest', 'principal') command = ["permission", "--role", role, "--principal", principal] self.noouttest(command) def promote_current_user(self): srcdir = self.config.get("broker", "srcdir") add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py") env = os.environ.copy() env['AQDCONF'] = self.config.baseconfig p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env) (out, err) = p.communicate() self.assertEqual(p.returncode, 0, "Failed to restore admin privs '%s', '%s'." % (out, err))
def makeService(self, options): # Start up coverage ASAP. coverage_dir = options["coveragedir"] if coverage_dir: os.makedirs(coverage_dir, 0755) if options["coveragerc"]: coveragerc = options["coveragerc"] else: coveragerc = None self.coverage = coverage.coverage(config_file=coveragerc) self.coverage.erase() self.coverage.start() # Get the config object. config = Config(configfile=options["config"]) # Helper for finishing off the coverage report. def stop_coverage(): log.msg("Finishing coverage") self.coverage.stop() aquilon_srcdir = os.path.join(config.get("broker", "srcdir"), "lib", "python2.6", "aquilon") sourcefiles = [] for dirpath, dirnames, filenames in os.walk(aquilon_srcdir): # FIXME: try to do this from the coverage config file if dirpath.endswith("aquilon"): dirnames.remove("client") elif dirpath.endswith("aqdb"): dirnames.remove("utils") for filename in filenames: if not filename.endswith('.py'): continue sourcefiles.append(os.path.join(dirpath, filename)) self.coverage.html_report(sourcefiles, directory=coverage_dir) self.coverage.xml_report(sourcefiles, outfile=os.path.join( coverage_dir, "aqd.xml")) with open(os.path.join(coverage_dir, "aqd.coverage"), "w") as outfile: self.coverage.report(sourcefiles, file=outfile) # Make sure the coverage report gets generated. if coverage_dir: reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage) # Set up the environment... m = Modulecmd() log_module_load(m, config.get("broker", "CheckNet_module")) if config.has_option("database", "module"): log_module_load(m, config.get("database", "module")) sys.path.append(config.get("protocols", "directory")) # Set this up before the aqdb libs get imported... integrate_logging(config) progname = os.path.split(sys.argv[0])[1] if progname == 'aqd': if config.get('broker', 'mode') != 'readwrite': log.msg("Broker started with aqd symlink, " "setting config mode to readwrite") config.set('broker', 'mode', 'readwrite') if progname == 'aqd_readonly': if config.get('broker', 'mode') != 'readonly': log.msg("Broker started with aqd_readonly symlink, " "setting config mode to readonly") config.set('broker', 'mode', 'readonly') log.msg("Loading broker in mode %s" % config.get('broker', 'mode')) # Dynamic import means that we can parse config options before # importing aqdb. This is a hack until aqdb can be imported without # firing up database connections. resources = __import__("aquilon.worker.resources", globals(), locals(), ["RestServer"], -1) RestServer = getattr(resources, "RestServer") restServer = RestServer(config) openSite = AnonSite(restServer) # twisted is nicely changing the umask for us when the process is # set to daemonize. This sets it back. restServer.set_umask() reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask) reactor.addSystemEventTrigger('after', 'startup', restServer.set_thread_pool_size) sockdir = config.get("broker", "sockdir") if not os.path.exists(sockdir): os.makedirs(sockdir, 0700) os.chmod(sockdir, 0700) if options["usesock"]: return strports.service("unix:%s/aqdsock" % sockdir, openSite) openport = config.get("broker", "openport") if config.has_option("broker", "bind_address"): bind_address = config.get("broker", "bind_address").strip() openaddr = "tcp:%s:interface=%s" % (openport, bind_address) else: # pragma: no cover bind_address = None openaddr = "tcp:%s" % openport # Return before firing up knc. if options["noauth"]: return strports.service(openaddr, openSite) sockname = os.path.join(sockdir, "kncsock") # This flag controls whether or not this process will start up # and monitor knc. Except for noauth mode knc has to be running, # but this process doesn't have to be the thing that starts it up. if config.getboolean("broker", "run_knc") or \ config.getboolean("broker", "run_git_daemon"): mon = GracefulProcessMonitor() # FIXME: Should probably run krb5_keytab here as well. # and/or verify that the keytab file exists. if config.getboolean("broker", "run_knc"): keytab = config.get("broker", "keytab") knc_args = [ "/usr/bin/env", "KRB5_KTNAME=FILE:%s" % keytab, config.get("kerberos", "knc"), "-lS", sockname ] if bind_address: knc_args.append("-a") knc_args.append(bind_address) knc_args.append(config.get("broker", "kncport")) mon.addProcess("knc", knc_args) if config.getboolean("broker", "run_git_daemon"): # The git daemon *must* be invoked using the form 'git-daemon' # instead of invoking git with a 'daemon' argument. The latter # will fork and exec git-daemon, resulting in a new pid that # the process monitor won't know about! gitpath = config.get("broker", "git_path") gitdaemon = config.get("broker", "git_daemon") ospath = os.environ.get("PATH", "") args = [ "/usr/bin/env", "PATH=%s:%s" % (gitpath, ospath), gitdaemon, "--export-all", "--base-path=%s" % config.get("broker", "git_daemon_basedir") ] if config.has_option("broker", "git_port"): args.append("--port=%s" % config.get("broker", "git_port")) if bind_address: args.append("--listen=%s" % bind_address) args.append(config.get("broker", "kingdir")) mon.addProcess("git-daemon", args) mon.startService() reactor.addSystemEventTrigger('before', 'shutdown', mon.stopService) # This socket is created by twisted and only accessed by knc as # connections come in. if os.path.exists(sockname): try: log.msg("Attempting to remove old socket '%s'" % sockname) os.remove(sockname) log.msg("Succeeded removing old socket.") except OSError, e: log.msg("Could not remove old socket '%s': %s" % (sockname, e))