def sync_domain(dbdomain, logger=LOGGER, locked=False):
"""Update templates on disk to match contents of branch in template-king.
If this domain is tracking another, first update the branch in
template-king with the latest from the tracking branch. Also save
the current (previous) commit as a potential rollback point.
"""
config = Config()
kingdir = config.get("broker", "kingdir")
domaindir = os.path.join(config.get("broker", "domainsdir"), dbdomain.name)
git_env = {"PATH": "%s:%s" % (config.get("broker", "git_path"),
os.environ.get("PATH", ""))}
if dbdomain.tracked_branch:
# Might need to revisit if using this helper from rollback...
run_command(["git", "push", ".",
"%s:%s" % (dbdomain.tracked_branch.name, dbdomain.name)],
path=kingdir, env=git_env, logger=logger)
run_command(["git", "fetch", "--prune"], path=domaindir, env=git_env, logger=logger)
if dbdomain.tracked_branch:
out = run_command(["git", "rev-list", "-n", "1", "HEAD"],
path=domaindir, env=git_env, logger=logger)
rollback_commit = out.strip()
try:
if not locked:
key = CompileKey(domain=dbdomain.name, logger=logger)
lock_queue.acquire(key)
run_command(["git", "reset", "--hard", "origin/%s" % dbdomain.name],
path=domaindir, env=git_env, logger=logger)
finally:
if not locked:
lock_queue.release(key)
if dbdomain.tracked_branch:
dbdomain.rollback_commit = rollback_commit
def run_git(args, env=None, path=".",
logger=LOGGER, loglevel=logging.INFO, filterre=None):
config = Config()
if env:
git_env = env.copy()
else:
git_env = {}
env_path = git_env.get("PATH", os.environ.get("PATH", ""))
git_env["PATH"] = "%s:%s" % (config.get("broker", "git_path"), env_path)
for name in ["git_author_name", "git_author_email",
"git_committer_name", "git_committer_email"]:
if not config.has_option("broker", name):
continue
value = config.get("broker", name)
git_env[name.upper()] = value
if isinstance(args, list):
git_args = args[:]
if git_args[0] != "git":
git_args.insert(0, "git")
else:
git_args = ["git", args]
return run_command(git_args, env=git_env, path=path,
logger=logger, loglevel=loglevel, filterre=filterre)
def testclonetemplateking(self):
config = Config()
source = config.get("unittest", "template_base")
dest = config.get("broker", "kingdir")
p = Popen(("/bin/rm", "-rf", dest), stdout=1, stderr=2)
rc = p.wait()
self.assertEqual(
rc, 0, "Failed to clear old template-king directory '%s'" % dest)
env = {}
env["PATH"] = "%s:%s" % (config.get(
"broker", "git_path"), os.environ.get("PATH", ""))
p = Popen(("git", "clone", "--bare", source, dest),
env=env,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code for clone of template-king, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (out, err))
# This value can be used to test against a different branch/commit
# than the current 'prod'.
new_prod = None
if config.has_option("unittest", "template_alternate_prod"):
new_prod = config.get("unittest", "template_alternate_prod")
if new_prod:
for domain in ['prod', 'ny-prod']:
p = Popen(("git", "push", ".", '+%s:%s' % (new_prod, domain)),
env=env,
cwd=dest,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code while setting alternate "
"'%s' branch locally to '%s':"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(domain, new_prod, out, err))
# Set the default branch
p = Popen(("git", "symbolic-ref", "HEAD", "refs/heads/prod"),
env=env,
cwd=dest,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0, "Non-zero return code while setting HEAD "
"to refs/heads/prod:"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n" % (out, err))
return
def outputdirs(self):
"""Returns a list of directories that should exist before compiling"""
config = Config()
dirs = []
dirs.append(config.get("broker", "profilesdir"))
# The regression tests occasionally have issues with panc
# auto-creating this directory - not sure why.
if self.domain.clusters:
dirs.append(os.path.join(config.get("broker", "quattordir"), "build", "xml", self.domain.name, "clusters"))
return dirs
def outputdirs(self):
"""Returns a list of directories that should exist before compiling"""
config = Config()
dirs = []
dirs.append(config.get("broker", "profilesdir"))
# The regression tests occasionally have issues with panc
# auto-creating this directory - not sure why.
if self.domain.clusters:
dirs.append(os.path.join(config.get("broker", "quattordir"),
"build", "xml", self.domain.name,
"clusters"))
return dirs
def testdisabletemplatetests(self):
config = Config()
kingdir = config.get("broker", "kingdir")
rundir = config.get("broker", "rundir")
env = {}
env["PATH"] = "%s:%s" % (config.get(
"broker", "git_path"), os.environ.get("PATH", ""))
tempdir = mkdtemp(prefix="fixup", dir=rundir)
p = Popen(("git", "clone", "--shared", kingdir, "template-king",
"--branch", "prod"),
cwd=tempdir,
env=env,
stdout=PIPE,
stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0, "Failed to clone template-king")
repodir = os.path.join(tempdir, "template-king")
makefile = os.path.join(repodir, "Makefile")
if os.path.exists(os.path.join(repodir, "t", "Makefile")):
p = Popen(("git", "rm", "-f", os.path.join("t", "Makefile")),
cwd=repodir,
env=env,
stdout=PIPE,
stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0, "Failed to remove t/Makefile")
p = Popen(("git", "commit", "-m", "Removed t/Makefile"),
cwd=repodir,
env=env,
stdout=PIPE,
stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0,
"Failed to commit removal of t/Makefile")
for branch in ['prod', 'ny-prod']:
p = Popen(("git", "push", "origin", "prod:%s" % branch),
cwd=repodir,
env=env,
stdout=PIPE,
stderr=PIPE)
out, err = p.communicate()
self.assertEqual(
p.returncode, 0, "Failed to push to %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(branch, out, err))
p = Popen(("rm", "-rf", tempdir))
p.communicate()
def directories(self):
"""Return a list of directories required for compiling this domain"""
config = Config()
dirs = []
if self.domain.branch_type == "domain":
dirs.append(os.path.join(config.get("broker", "domainsdir"), self.domain.name))
dirs.append(os.path.join(config.get("broker", "quattordir"), "cfg", "domains", self.domain.name))
dirs.append(os.path.join(config.get("broker", "quattordir"), "build", "xml", self.domain.name))
return dirs
def sync_domain(dbdomain, logger=LOGGER, locked=False):
"""Update templates on disk to match contents of branch in template-king.
If this domain is tracking another, first update the branch in
template-king with the latest from the tracking branch. Also save
the current (previous) commit as a potential rollback point.
"""
config = Config()
session = object_session(dbdomain)
kingdir = config.get("broker", "kingdir")
domaindir = os.path.join(config.get("broker", "domainsdir"), dbdomain.name)
git_env = {
"PATH":
"%s:%s" %
(config.get("broker", "git_path"), os.environ.get("PATH", ""))
}
if dbdomain.tracked_branch:
# Might need to revisit if using this helper from rollback...
run_command([
"git", "push", ".",
"%s:%s" % (dbdomain.tracked_branch.name, dbdomain.name)
],
path=kingdir,
env=git_env,
logger=logger)
run_command(["git", "fetch", "--prune"],
path=domaindir,
env=git_env,
logger=logger)
if dbdomain.tracked_branch:
out = run_command(["git", "rev-list", "-n", "1", "HEAD"],
path=domaindir,
env=git_env,
logger=logger)
rollback_commit = out.strip()
try:
if not locked:
key = CompileKey(domain=dbdomain.name, logger=logger)
lock_queue.acquire(key)
run_command(["git", "reset", "--hard",
"origin/%s" % dbdomain.name],
path=domaindir,
env=git_env,
logger=logger)
finally:
if not locked:
lock_queue.release(key)
if dbdomain.tracked_branch:
dbdomain.rollback_commit = rollback_commit
session.add(dbdomain)
def testclonetemplateking(self):
config = Config()
source = config.get("unittest", "template_base")
dest = config.get("broker", "kingdir")
p = Popen(("/bin/rm", "-rf", dest), stdout=1, stderr=2)
rc = p.wait()
self.assertEqual(rc, 0,
"Failed to clear old template-king directory '%s'" %
dest)
env = {}
env["PATH"] = "%s:%s" % (config.get("broker", "git_path"),
os.environ.get("PATH", ""))
p = Popen(("git", "clone", "--bare", source, dest),
env=env, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code for clone of template-king, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (out, err))
# This value can be used to test against a different branch/commit
# than the current 'prod'.
new_prod = None
if config.has_option("unittest", "template_alternate_prod"):
new_prod = config.get("unittest", "template_alternate_prod")
if new_prod:
for domain in ['prod', 'ny-prod']:
p = Popen(("git", "push", ".", '+%s:%s' % (new_prod, domain)),
env=env, cwd=dest, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code while setting alternate "
"'%s' branch locally to '%s':"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (domain, new_prod, out, err))
# Set the default branch
p = Popen(("git", "symbolic-ref", "HEAD", "refs/heads/prod"),
env=env, cwd=dest, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Non-zero return code while setting HEAD "
"to refs/heads/prod:"
"\nSTDOUT:\n@@@\n'%s'\n@@@\n"
"\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (out, err))
return
def config_proto(self, node, command):
desc_node = node.find("message_class")
if desc_node is None or "name" not in desc_node.attrib or \
"module" not in desc_node.attrib:
raise ProtocolError("Invalid protobuf definition for %s." % command)
module = desc_node.attrib["module"]
msgclass = desc_node.attrib["name"]
if module in self.loaded_protocols and \
self.loaded_protocols[module] == False:
raise ProtocolError("Protocol %s: previous import attempt was "
"unsuccessful" % module)
if module not in self.loaded_protocols:
config = Config()
protodir = config.get("protocols", "directory")
# Modifying sys.path here is ugly. We could try playing with
# find_module()/load_module(), but there are dependencies between
# the protocols, that could fail if sys.path is not set up and the
# protocols are loaded in the wrong order.
if protodir not in sys.path:
sys.path.append(protodir)
try:
self.loaded_protocols[module] = __import__(module)
except ImportError, err: # pragma: no cover
self.loaded_protocols[module] = False
raise ProtocolError("Protocol %s: %s" % (module, err))
def discover_network_types(dbapi_con, connection_record): # pylint: disable=W0613
config = Config()
if not config.has_option("broker", "default_network_type"): # pragma: no cover
raise InternalError("The default_network_type option is missing from "
"the [broker] section in the configuration.")
default_type = config.get("broker", "default_network_type")
default_section = "network_" + default_type
if not config.has_section(default_section): # pragma: no cover
raise InternalError("The default network type is %s, but there's no "
"section named [%s] in the configuration." %
(default_type, default_section))
nettypes = {}
# This function should be called only once, but you never know...
if Network.network_type_map:
return
for section in config.sections():
if not section.startswith("network_"):
continue
name = section[8:]
nettypes[name] = NetworkProperties(config, name)
LOGGER.info("Configured network type %s" % name)
Network.network_type_map = nettypes
Network.default_network_props = nettypes[default_type]
def discover_network_types(dbapi_con, connection_record): # pylint: disable=W0613
config = Config()
if not config.has_option("broker", "default_network_type"): # pragma: no cover
raise InternalError("The default_network_type option is missing from "
"the [broker] section in the configuration.")
default_type = config.get("broker", "default_network_type")
default_section = "network_" + default_type
if not config.has_section(default_section): # pragma: no cover
raise InternalError("The default network type is %s, but there's no "
"section named [%s] in the configuration." %
(default_type, default_section))
nettypes = {}
# This function should be called only once, but you never know...
if Network.network_type_map:
return
for section in config.sections():
if not section.startswith("network_"):
continue
name = section[8:]
nettypes[name] = NetworkProperties(config, name)
LOGGER.info("Configured network type %s", name)
Network.network_type_map = nettypes
Network.default_network_props = nettypes[default_type]
def compile(self, session, only=None, locked=False,
panc_debug_include=None, panc_debug_exclude=None,
cleandeps=False):
"""The build directories are checked and constructed
if necessary, so no prior setup is required. The compile may
take some time (current rate is 10 hosts per second, with a
couple of seconds of constant overhead), and the possibility
of blocking on the compile lock.
If the 'only' parameter is provided, then it should be a
list or set containing the profiles that need to be compiled.
May raise ArgumentError exception, else returns the standard
output (as a string) of the compile
"""
config = Config()
if self.domain.branch_type == 'sandbox':
if not self.author:
raise InternalError("Missing required author to compile "
"sandbox %s" % self.domain.name)
sandboxdir = os.path.join(config.get("broker", "templatesdir"),
self.author.name, self.domain.name)
if not os.path.exists(sandboxdir):
raise ArgumentError("Sandbox directory '%s' does not exist." %
sandboxdir)
if not self.sandbox_has_latest(config, sandboxdir):
self.logger.warn("Sandbox %s/%s does not contain the "
"latest changes from the prod domain. If "
"there are failures try "
"`git fetch && git merge origin/prod`" %
(self.author.name, self.domain.name))
self.logger.info("preparing domain %s for compile" % self.domain.name)
# Ensure that the compile directory is in a good state.
outputdir = config.get("broker", "profilesdir")
for d in self.directories() + self.outputdirs():
if not os.path.exists(d):
try:
self.logger.info("creating %s" % d)
os.makedirs(d)
except OSError, e:
raise ArgumentError("Failed to mkdir %s: %s" % (d, e))
def __init__(self, logger=LOGGER):
config = Config()
self.logger = logger
self.dsdb = config.get("broker", "dsdb")
self.dsdb_use_testdb = config.getboolean("broker", "dsdb_use_testdb")
self.location_sync = config.getboolean("broker", "dsdb_location_sync")
self.actions = []
self.rollback_list = []
def __init__(self, logger=LOGGER):
config = Config()
self.logger = logger
self.dsdb = config.get("broker", "dsdb")
self.dsdb_use_testdb = config.getboolean("broker", "dsdb_use_testdb")
self.location_sync = config.getboolean("broker", "dsdb_location_sync")
self.actions = []
self.rollback_list = []
def compile(
self, session, only=None, locked=False, panc_debug_include=None, panc_debug_exclude=None, cleandeps=False
):
"""The build directories are checked and constructed
if necessary, so no prior setup is required. The compile may
take some time (current rate is 10 hosts per second, with a
couple of seconds of constant overhead), and the possibility
of blocking on the compile lock.
If the 'only' parameter is provided, then it should be a
list or set containing the profiles that need to be compiled.
May raise ArgumentError exception, else returns the standard
output (as a string) of the compile
"""
config = Config()
if self.domain.branch_type == "sandbox":
if not self.author:
raise InternalError("Missing required author to compile " "sandbox %s" % self.domain.name)
sandboxdir = os.path.join(config.get("broker", "templatesdir"), self.author.name, self.domain.name)
if not os.path.exists(sandboxdir):
raise ArgumentError("Sandbox directory '%s' does not exist." % sandboxdir)
if not self.sandbox_has_latest(config, sandboxdir):
self.logger.warn(
"Sandbox %s/%s does not contain the "
"latest changes from the prod domain. If "
"there are failures try "
"`git fetch && git merge origin/prod`" % (self.author.name, self.domain.name)
)
self.logger.info("preparing domain %s for compile" % self.domain.name)
# Ensure that the compile directory is in a good state.
outputdir = config.get("broker", "profilesdir")
for d in self.directories() + self.outputdirs():
if not os.path.exists(d):
try:
self.logger.info("creating %s" % d)
os.makedirs(d)
except OSError, e:
raise ArgumentError("Failed to mkdir %s: %s" % (d, e))
def cache_storage_data(only=None):
"""
Scan a storeng-style data file, checking each line as we go
Storeng-style data files are blocks of data. Each block starts
with a comment describing the fields for all subsequent lines. A
block can start at any time. Fields are separated by '|'.
This function will invoke the function after parsing every data
line. The function will be called with a dict of the fields. If the
function returns True, then we stop scanning the file, else we continue
on until there is nothing left to parse.
dbshare can be a Share
"""
config = Config()
sharedata = {}
found_header = False
header_idx = {}
with open(config.get("broker", "sharedata")) as datafile:
for line in datafile:
if line[0] == '#':
# A header line
found_header = True
hdr = line[1:].rstrip().split('|')
header_idx = {}
for idx, name in enumerate(hdr):
header_idx[name] = idx
# Silently discard lines that don't have all the required info
for k in ["objtype", "pshare", "server", "dg"]:
if k not in header_idx:
found_header = False
elif not found_header:
# We haven't found the right header line
continue
else:
fields = line.rstrip().split('|')
if len(fields) != len(header_idx): # Silently ignore invalid lines
continue
if fields[header_idx["objtype"]] != "pshare":
continue
sharedata[fields[header_idx["pshare"]]] = ShareInfo(
server=fields[header_idx["server"]],
mount="/vol/%s/%s" % (fields[header_idx["dg"]],
fields[header_idx["pshare"]])
)
# Take a shortcut if we need just a single entry
if only and only == fields[header_idx["pshare"]]:
break
return sharedata
def _snapshot_db(self, test):
# If there was an error, and we're using SQLite, create a snapshot
# TODO: create a git-managed snapshot of the plenaries/profiles as well
config = Config()
dsn = config.get("database", "dsn")
if dsn.startswith("sqlite:///"):
dbfile = dsn[10:]
target = dbfile + ".%s:%s" % (test.__class__.__name__,
test._testMethodName)
call(["/bin/cp", "-a", dbfile, target])
def directories(self):
"""Return a list of directories required for compiling this domain"""
config = Config()
dirs = []
if self.domain.branch_type == 'domain':
dirs.append(os.path.join(config.get("broker", "domainsdir"),
self.domain.name))
dirs.append(os.path.join(config.get("broker", "quattordir"),
"cfg",
"domains",
self.domain.name))
dirs.append(os.path.join(config.get("broker", "quattordir"),
"build",
"xml",
self.domain.name))
return dirs
def testcloneswrep(self):
config = Config()
source = config.get("unittest", "swrep_repository")
dest = os.path.join(config.get("broker", "swrepdir"), "repository")
p = Popen(("/bin/rm", "-rf", dest), stdout=1, stderr=2)
rc = p.wait()
self.assertEqual(rc, 0,
"Failed to clear old swrep directory '%s'" %
dest)
env = {}
env["PATH"] = "%s:%s" % (config.get("broker", "git_path"),
os.environ.get("PATH", ""))
p = Popen(("git", "clone", source, dest),
env=env, stdout=PIPE, stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code for clone of swrep, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (out, err))
return
def testcloneswrep(self):
config = Config()
source = config.get("unittest", "swrep_repository")
dest = os.path.join(config.get("broker", "swrepdir"), "repository")
p = Popen(("/bin/rm", "-rf", dest), stdout=1, stderr=2)
rc = p.wait()
self.assertEqual(rc, 0,
"Failed to clear old swrep directory '%s'" % dest)
env = {}
env["PATH"] = "%s:%s" % (config.get(
"broker", "git_path"), os.environ.get("PATH", ""))
p = Popen(("git", "clone", source, dest),
env=env,
stdout=PIPE,
stderr=PIPE)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0, "Non-zero return code for clone of swrep, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" % (out, err))
return
def directories(self):
"""Return a list of directories required for compiling this domain"""
config = Config()
dirs = []
if self.domain.branch_type == 'domain':
dirs.append(os.path.join(config.get("broker", "domainsdir"),
self.domain.name))
dirs.append(os.path.join(config.get("broker", "cfgdir"),
"domains", self.domain.name))
# This is a bit redundant. When creating the directories, the "clusters"
# subdir would be enough; when removing them, the base dir would be
# enough. Having both does not hurt and does not need such extra logic.
dirs.append(os.path.join(config.get("broker", "quattordir"),
"build", self.domain.name))
dirs.append(os.path.join(config.get("broker", "quattordir"),
"build", self.domain.name, "clusters"))
return dirs
def main():
parser = argparse.ArgumentParser(description="Send out broker notifications")
parser.add_argument("-c", "--config", dest="config",
help="location of the broker configuration file")
parser.add_argument("--one_shot", action="store_true",
help="do just a single run and then exit")
parser.add_argument("--debug", action="store_true",
help="turn on debug logs on stderr")
opts = parser.parse_args()
config = Config(configfile=opts.config)
# These modules must be imported after the configuration has been
# initialized
from aquilon.aqdb.db_factory import DbFactory
db = DbFactory()
if opts.debug:
level = logging.DEBUG
logging.basicConfig(level=level, stream=sys.stderr,
format='%(asctime)s [%(levelname)s] %(message)s')
else:
level = logging.INFO
logfile = os.path.join(config.get("broker", "logdir"), "aq_notifyd.log")
handler = WatchedFileHandler(logfile)
handler.setLevel(level)
formatter = logging.Formatter('%(asctime)s [%(levelname)s] %(message)s')
handler.setFormatter(formatter)
rootlog = logging.getLogger()
rootlog.addHandler(handler)
rootlog.setLevel(level)
# Apply configured log settings
for logname, level in config.items("logging"):
if level not in logging._levelNames:
continue
logging.getLogger(logname).setLevel(logging._levelNames[level])
logger = logging.getLogger("aq_notifyd")
if opts.one_shot:
update_index_and_notify(config, logger, db)
else:
signal.signal(signal.SIGTERM, exit_handler)
signal.signal(signal.SIGINT, exit_handler)
run_loop(config, logger, db)
def testdisabletemplatetests(self):
config = Config()
kingdir = config.get("broker", "kingdir")
rundir = config.get("broker", "rundir")
env = {}
env["PATH"] = "%s:%s" % (config.get("broker", "git_path"),
os.environ.get("PATH", ""))
tempdir = mkdtemp(prefix="fixup", dir=rundir)
p = Popen(("git", "clone", "--shared", kingdir, "template-king",
"--branch", "prod"),
cwd=tempdir, env=env, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0, "Failed to clone template-king")
repodir = os.path.join(tempdir, "template-king")
makefile = os.path.join(repodir, "Makefile")
if os.path.exists(os.path.join(repodir, "t", "Makefile")):
p = Popen(("git", "rm", "-f", os.path.join("t", "Makefile")),
cwd=repodir, env=env, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0, "Failed to remove t/Makefile")
p = Popen(("git", "commit", "-m", "Removed t/Makefile"),
cwd=repodir, env=env, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0, "Failed to commit removal of t/Makefile")
for branch in ['prod', 'ny-prod']:
p = Popen(("git", "push", "origin", "prod:%s" % branch),
cwd=repodir, env=env, stdout=PIPE, stderr=PIPE)
out, err = p.communicate()
self.assertEqual(p.returncode, 0,
"Failed to push to %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (branch, out, err))
p = Popen(("rm", "-rf", tempdir))
p.communicate()
def run_git(args,
env=None,
path=".",
logger=LOGGER,
loglevel=logging.INFO,
filterre=None):
config = Config()
if env:
git_env = env.copy()
else:
git_env = {}
env_path = git_env.get("PATH", os.environ.get("PATH", ""))
git_env["PATH"] = "%s:%s" % (config.get("broker", "git_path"), env_path)
for name in [
"git_author_name", "git_author_email", "git_committer_name",
"git_committer_email"
]:
if not config.has_option("broker", name):
continue
value = config.get("broker", name)
git_env[name.upper()] = value
if isinstance(args, list):
git_args = args[:]
if git_args[0] != "git":
git_args.insert(0, "git")
else:
git_args = ["git", args]
return run_command(git_args,
env=git_env,
path=path,
logger=logger,
loglevel=loglevel,
filterre=filterre)
def __init__(self, network=None, network_type=None, **kw):
# pylint: disable=W0621
if not isinstance(network, IPv4Network):
raise InternalError("Expected an IPv4Network, got: %s" %
type(network))
if not network_type:
config = Config()
network_type = config.get("broker", "default_network_type")
self._network = network
self._props = self.network_type_map.get(self.network_type,
self.default_network_props)
super(Network, self).__init__(ip=network.network,
cidr=network.prefixlen,
network_type=network_type, **kw)
def __init__(self, network=None, network_type=None, **kw):
# pylint: disable=W0621
if not isinstance(network, IPv4Network):
raise InternalError("Expected an IPv4Network, got: %s" %
type(network))
if not network_type:
config = Config()
network_type = config.get("broker", "default_network_type")
self._network = network
self._props = self.network_type_map.get(self.network_type,
self.default_network_props)
super(Network, self).__init__(ip=network.network,
cidr=network.prefixlen,
network_type=network_type, **kw)
def main():
from aquilon.config import Config
config = Config()
if config.has_option("database", "module"):
ms.modulecmd.load(config.get("database", "module"))
db = DbFactory()
Base.metadata.bind = db.engine
session = db.Session()
add_interfaces(session)
add_addresses(session)
session.rollback()
raise Exception("Replace the rollback() in the code with commit() when "
"ready to go, and disable this exception")
def main():
from aquilon.config import Config
config = Config()
if config.has_option("database", "module"):
ms.modulecmd.load(config.get("database", "module"))
db = DbFactory()
Base.metadata.bind = db.engine
session = db.Session()
add_interfaces(session)
add_addresses(session)
session.rollback()
raise Exception("Replace the rollback() in the code with commit() when "
"ready to go, and disable this exception")
def teststop(self):
config = Config()
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
self.assert_(os.path.exists(pidfile))
f = file(pidfile)
pid = f.readline()
self.assertNotEqual(pid, "")
f.close()
pid = int(pid)
os.kill(pid, signal.SIGTERM)
# Wait for the broker to shut down. E.g. generating code coverage may
# take some time.
i = 0
while i < 180:
i += 1
try:
os.kill(pid, 0)
except OSError:
break
sleep(1)
# Verify that the broker is down
self.failUnlessRaises(OSError, os.kill, pid, 0)
def teststop(self):
config = Config()
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
self.assert_(os.path.exists(pidfile))
f = file(pidfile)
pid = f.readline()
self.assertNotEqual(pid, "")
f.close()
pid = int(pid)
os.kill(pid, signal.SIGTERM)
# Wait for the broker to shut down. E.g. generating code coverage may
# take some time.
i = 0
while i < 180:
i += 1
try:
os.kill(pid, 0)
except OSError:
break
sleep(1)
# Verify that the broker is down
self.failUnlessRaises(OSError, os.kill, pid, 0)
def teststart(self):
# FIXME: Either remove any old pidfiles, or ignore it as a warning
# from stderr... or IMHO (daqscott) if pid files exist and are knc or
# python processes, kill -9 the pids and delete the files (with a
# warning message it tickles you)
config = Config()
twistd = os.path.join(config.get("broker", "srcdir"), "lib",
"python2.6", "aquilon", "unittest_patches.py")
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
logfile = config.get("broker", "logfile")
# Specify twistd and options...
args = [
sys.executable, twistd, "--pidfile", pidfile, "--logfile", logfile
]
if config.has_option("unittest", "profile"):
if config.getboolean("unittest", "profile"):
args.append("--profile")
args.append(
os.path.join(config.get("broker", "logdir"),
"aqd.profile"))
args.append("--profiler=cProfile")
args.append("--savestats")
# And then aqd and options...
args.extend(["aqd", "--config", config.baseconfig])
if config.has_option("unittest", "coverage"):
if config.getboolean("unittest", "coverage"):
args.append("--coveragedir")
dir = os.path.join(config.get("broker", "logdir"), "coverage")
args.append(dir)
coveragerc = os.path.join(config.get("broker", "srcdir"),
"tests", "coverage.rc")
args.append("--coveragerc")
args.append(coveragerc)
p = Popen(args)
self.assertEqual(p.wait(), 0)
def teststart(self):
# FIXME: Either remove any old pidfiles, or ignore it as a warning
# from stderr... or IMHO (daqscott) if pid files exist and are knc or
# python processes, kill -9 the pids and delete the files (with a
# warning message it tickles you)
config = Config()
twistd = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon", "unittest_patches.py")
pidfile = os.path.join(config.get("broker", "rundir"), "aqd.pid")
logfile = config.get("broker", "logfile")
# Specify twistd and options...
args = [sys.executable, twistd,
"--pidfile", pidfile, "--logfile", logfile]
if config.has_option("unittest", "profile"):
if config.getboolean("unittest", "profile"):
args.append("--profile")
args.append(os.path.join(config.get("broker", "logdir"),
"aqd.profile"))
args.append("--profiler=cProfile")
args.append("--savestats")
# And then aqd and options...
args.extend(["aqd", "--config", config.baseconfig])
if config.has_option("unittest", "coverage"):
if config.getboolean("unittest", "coverage"):
args.append("--coveragedir")
dir = os.path.join(config.get("broker", "logdir"), "coverage")
args.append(dir)
coveragerc = os.path.join(config.get("broker", "srcdir"),
"tests", "coverage.rc")
args.append("--coveragerc")
args.append(coveragerc)
p = Popen(args)
self.assertEqual(p.wait(), 0)
def makeService(self, options):
# Start up coverage ASAP.
coverage_dir = options["coveragedir"]
if coverage_dir:
os.makedirs(coverage_dir, 0755)
if options["coveragerc"]:
coveragerc = options["coveragerc"]
else:
coveragerc = None
self.coverage = coverage.coverage(config_file=coveragerc)
self.coverage.erase()
self.coverage.start()
# Get the config object.
config = Config(configfile=options["config"])
# Helper for finishing off the coverage report.
def stop_coverage():
log.msg("Finishing coverage")
self.coverage.stop()
aquilon_srcdir = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon")
sourcefiles = []
for dirpath, dirnames, filenames in os.walk(aquilon_srcdir):
# FIXME: try to do this from the coverage config file
if dirpath.endswith("aquilon"):
dirnames.remove("client")
elif dirpath.endswith("aqdb"):
dirnames.remove("utils")
for filename in filenames:
if not filename.endswith('.py'):
continue
sourcefiles.append(os.path.join(dirpath, filename))
self.coverage.html_report(sourcefiles, directory=coverage_dir)
self.coverage.xml_report(sourcefiles,
outfile=os.path.join(coverage_dir, "aqd.xml"))
with open(os.path.join(coverage_dir, "aqd.coverage"), "w") as outfile:
self.coverage.report(sourcefiles, file=outfile)
# Make sure the coverage report gets generated.
if coverage_dir:
reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage)
# Set up the environment...
m = Modulecmd()
log_module_load(m, config.get("broker", "CheckNet_module"))
if config.has_option("database", "module"):
log_module_load(m, config.get("database", "module"))
sys.path.append(config.get("protocols", "directory"))
# Set this up before the aqdb libs get imported...
integrate_logging(config)
progname = os.path.split(sys.argv[0])[1]
if progname == 'aqd':
if config.get('broker', 'mode') != 'readwrite':
log.msg("Broker started with aqd symlink, "
"setting config mode to readwrite")
config.set('broker', 'mode', 'readwrite')
if progname == 'aqd_readonly':
if config.get('broker', 'mode') != 'readonly':
log.msg("Broker started with aqd_readonly symlink, "
"setting config mode to readonly")
config.set('broker', 'mode', 'readonly')
log.msg("Loading broker in mode %s" % config.get('broker', 'mode'))
# Dynamic import means that we can parse config options before
# importing aqdb. This is a hack until aqdb can be imported without
# firing up database connections.
resources = __import__("aquilon.worker.resources", globals(), locals(),
["RestServer"], -1)
RestServer = getattr(resources, "RestServer")
restServer = RestServer(config)
openSite = AnonSite(restServer)
# twisted is nicely changing the umask for us when the process is
# set to daemonize. This sets it back.
restServer.set_umask()
reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask)
reactor.addSystemEventTrigger('after', 'startup',
restServer.set_thread_pool_size)
sockdir = config.get("broker", "sockdir")
if not os.path.exists(sockdir):
os.makedirs(sockdir, 0700)
os.chmod(sockdir, 0700)
if options["usesock"]:
return strports.service("unix:%s/aqdsock" % sockdir, openSite)
openport = config.get("broker", "openport")
if config.has_option("broker", "bind_address"):
bind_address = config.get("broker", "bind_address").strip()
openaddr = "tcp:%s:interface=%s" % (openport, bind_address)
else: # pragma: no cover
bind_address = None
openaddr = "tcp:%s" % openport
# Return before firing up knc.
if options["noauth"]:
return strports.service(openaddr, openSite)
sockname = os.path.join(sockdir, "kncsock")
# This flag controls whether or not this process will start up
# and monitor knc. Except for noauth mode knc has to be running,
# but this process doesn't have to be the thing that starts it up.
if config.getboolean("broker", "run_knc") or \
config.getboolean("broker", "run_git_daemon"):
mon = GracefulProcessMonitor()
# FIXME: Should probably run krb5_keytab here as well.
# and/or verify that the keytab file exists.
if config.getboolean("broker", "run_knc"):
keytab = config.get("broker", "keytab")
knc_args = ["/usr/bin/env",
"KRB5_KTNAME=FILE:%s" % keytab,
config.get("kerberos", "knc"), "-lS", sockname]
if bind_address:
knc_args.append("-a")
knc_args.append(bind_address)
knc_args.append(config.get("broker", "kncport"))
mon.addProcess("knc", knc_args)
if config.getboolean("broker", "run_git_daemon"):
# The git daemon *must* be invoked using the form 'git-daemon'
# instead of invoking git with a 'daemon' argument. The latter
# will fork and exec git-daemon, resulting in a new pid that
# the process monitor won't know about!
gitpath = config.get("broker", "git_path")
gitdaemon = config.get("broker", "git_daemon")
ospath = os.environ.get("PATH", "")
args = ["/usr/bin/env", "PATH=%s:%s" % (gitpath, ospath),
gitdaemon, "--export-all", "--base-path=%s" %
config.get("broker", "git_daemon_basedir")]
if config.has_option("broker", "git_port"):
args.append("--port=%s" % config.get("broker", "git_port"))
if bind_address:
args.append("--listen=%s" % bind_address)
args.append(config.get("broker", "kingdir"))
mon.addProcess("git-daemon", args)
mon.startService()
reactor.addSystemEventTrigger('before', 'shutdown', mon.stopService)
# This socket is created by twisted and only accessed by knc as
# connections come in.
if os.path.exists(sockname):
try:
log.msg("Attempting to remove old socket '%s'" % sockname)
os.remove(sockname)
log.msg("Succeeded removing old socket.")
except OSError, e:
log.msg("Could not remove old socket '%s': %s" % (sockname, e))
nullable=False)
name = Column(String(255), nullable=False)
value = Column(String(255), default='True', nullable=False)
__table_args__ = (PrimaryKeyConstraint(xtn_id, name, value,
name="xtn_detail_pk"),
Index('xtn_dtl_name_idx', name,
oracle_compress=True),
Index('xtn_dtl_value_idx', value, oracle_compress=True),
{'oracle_compress': 'OLTP'})
Xtn.args = relationship(XtnDetail, lazy="joined", order_by=[XtnDetail.name])
if config.has_option('database', 'audit_schema'): # pragma: no cover
schema = config.get('database', 'audit_schema')
Xtn.__table__.schema = schema
XtnEnd.__table__.schema = schema
XtnDetail.__table__.schema = schema
def start_xtn(session, xtn_id, username, command, is_readonly, details, ignore):
""" Wrapper to log the start of a transaction (or running command).
Takes a dictionary with the transaction parameters. The keys are
command, usename, readonly, and details. The details parameter
is itself a a dictionary of option names to option values provided
for the command.
The options_to_split is a list of any options that need to be
split on newlines. Typically this is --list and/or --hostlist.
import os
import logging
from aquilon.exceptions_ import InternalError, IncompleteError
from aquilon.config import Config
from aquilon.worker.locks import lock_queue, CompileKey
from aquilon.worker.processes import write_file, read_file, remove_file
from aquilon.worker.templates.panutils import pan_assign, pan_variable
from mako.lookup import TemplateLookup
from aquilon.worker.formats.formatters import ObjectFormatter
LOGGER = logging.getLogger(__name__)
_config = Config()
TEMPLATE_EXTENSION = _config.get("panc", "template_extension")
class Plenary(object):
template_type = None
""" Specifies the PAN template type to generate """
handlers = {}
""" The handlers dictionary should have an entry for every subclass.
Typically this will be defined immediately after defining the
subclass.
"""
def __init__(self, dbobj=None, logger=LOGGER):
self.config = Config()
class AuthorizationBroker(object):
"""Handles any behind the scenes work in figuring out entitlements."""
# Borg
__shared_state = {}
def __init__(self):
self.__dict__ = self.__shared_state
self.config = Config()
def raise_auth_error(self, principal, action, resource):
auth_msg = self.config.get("broker", "authorization_error")
raise AuthorizationException("Unauthorized access attempt by %s to %s "
"on %s. %s" %
(principal, action, resource, auth_msg))
# FIXME: Hard coded check for now.
def check(self, principal, dbuser, action, resource):
if action.startswith('show') or action.startswith('search') \
or action.startswith('cat') or action == 'status' \
or action == 'dump_dns':
return True
if dbuser is None:
raise AuthorizationException(
"Unauthorized anonymous access attempt to %s on %s" %
(action, resource))
# Special-casing the aquilon hosts... this is a special user
# that provides a bucket for all host-generated activity.
if self._check_aquilonhost(principal, dbuser, action, resource):
return True
if dbuser.role.name == 'nobody':
self.raise_auth_error(principal, action, resource)
# Right now, anybody in a group can do anything they want, except...
if action in [
'add_archetype', 'update_archetype', 'del_archetype',
'add_vendor', 'del_vendor', 'add_os', 'update_os', 'del_os',
'add_model', 'update_model', 'del_model', 'add_organization',
'del_organization', 'add_grn', 'del_grn', 'update_grn',
'add_vlan', 'del_vlan', 'rollback', 'update_city'
]:
if dbuser.role.name not in ['engineering', 'aqd_admin']:
raise AuthorizationException(
"Must have the engineering or aqd_admin role to %s." %
action)
if action in ['permission']:
if dbuser.role.name not in ['aqd_admin', 'gatekeeper']:
raise AuthorizationException(
"Must have the gatekeeper or aqd_admin role to %s." %
action)
if action in ['flush', 'refresh_windows_hosts']:
if dbuser.role.name not in ['aqd_admin']:
raise AuthorizationException(
"Must have the aqd_admin role to %s." % action)
if dbuser.role.name == 'winops':
if action not in [
'add_host', 'add_windows_host', 'make_cluster',
'reconfigure', 'update_machine'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'winops_server':
# Only need add/update_cluster for hacluster VCenters
if action not in [
'add_windows_host', 'del_windows_host', 'add_aurora_host',
'add_alias', 'update_alias', 'del_alias', 'add_machine',
'add_host', 'add_interface_hostname',
'add_interface_machine', 'add_interface_address',
'add_address', 'del_address', 'reconfigure', 'add_cluster',
'make_cluster', 'update_cluster', 'change_status',
'change_status_cluster', 'add_service_instance',
'map_service', 'bind_server', 'update_machine'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'mssb_unixops':
if action not in [
'add_machine', 'add_interface_hostname',
'add_interface_machine', 'add_interface_address',
'add_alias', 'update_alias', 'del_alias', 'add_address',
'del_address', 'add_host', 'change_status'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'spot_server':
if action not in [
'add_machine', 'del_machine', 'add_machine_prefix',
'add_interface_hostname', 'add_interface_machine',
'add_interface_address', 'del_interface',
'del_interface_address', 'add_address', 'del_address',
'add_host', 'del_host', 'add_alias', 'del_alias', 'make',
'make_cluster', 'pxeswitch', 'change_status', 'add_room',
'add_rack', 'add_rack_room', 'add_disk', 'del_disk',
'del_disk_disk'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'resource_pool':
if action not in ['add_address', 'del_address']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'hpevelo':
if action not in [
'reconfigure', 'pxeswitch', 'change_status', 'add_disk',
'del_disk', 'del_disk_disk'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'location_admin':
if action not in [
'add_building', 'del_building', 'add_city', 'del_city'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'telco_operations':
if action not in [
'add_rack', 'add_switch', 'update_rack', 'update_switch',
'del_rack', 'del_switch', 'add_interface_switch',
'del_interface_switch', 'update_interface_switch',
'add_interface_address_switch',
'del_interface_address_switch', 'add_alias', 'del_alias',
'refresh_network', 'update_router'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'edc':
if action not in [
'add_rack', 'add_machine', 'add_host',
'add_interface_machine', 'add_interface_hostname',
'add_interface_address', 'add_alias', 'add_manager',
'update_rack', 'update_machine', 'update_alias',
'update_interface_hostname', 'update_interface_machine',
'del_rack', 'del_machine', 'del_host', 'del_interface',
'del_interface_address', 'del_alias', 'del_manager'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'maintech':
if action not in [
'pxeswitch', 'pxeswitch_list', 'compile',
'compile_hostname', 'change_status',
'update_interface_hostname', 'update_interface_machine'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'itsec':
if action not in [
'pxeswitch', 'pxeswitch_list', 'compile',
'compile_hostname', 'change_status', 'make', 'reconfigure',
'reconfigure_list', 'reconfigure_hostlist',
'add_interface_machine', 'add_interface_hostname',
'add_interface_address', 'update_interface_hostname',
'update_interface_machine', 'del_interface',
'del_interface_address', 'add_disk', 'del_disk',
'del_disk_disk', 'add_machine', 'del_machine',
'update_machine', 'update_machine_hostname', 'add_host',
'del_host', 'add_alias', 'add_manager', 'del_alias',
'del_manager'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'unixops_l2':
if action not in [
'add_host', 'add_windows_host', 'del_host',
'del_windows_host', 'compile', 'compile_hostname',
'compile_cluster', 'reconfigure', 'change_status',
'reconfigure_list', 'reconfigure_hostlist', 'pxeswitch',
'pxeswitch_list', 'add_interface_chassis',
'add_interface_hostname', 'add_interface_machine',
'add_interface_address', 'del_interface',
'del_interface_address', 'add_reboot_intervention',
'add_reboot_schedule', 'del_reboot_intervention',
'del_reboot_schedule', 'update_interface_hostname',
'update_interface_machine', 'add_machine', 'del_machine',
'update_machine', 'update_machine_hostname', 'add_cluster',
'del_cluster', 'add_esx_cluster', 'update_esx_cluster',
'del_esx_cluster', 'bind_esx_cluster_hostname',
'rebind_esx_cluster_hostname', 'cluster', 'uncluster',
'change_status_cluster', 'add_manager',
'add_dynamic_range', 'add_disk', 'add_auxiliary',
'del_manager', 'del_auxiliary', 'del_disk',
'del_disk_disk', 'add_service_instance',
'update_service_instance', 'del_service_instance',
'add_alias', 'del_alias', 'add_filesystem',
'del_filesystem', 'poll_switch', 'poll_switch_switch',
'add_rack', 'add_rack_room', 'add_chassis', 'del_rack',
'del_chassis', 'add_bunker', 'del_bunker', 'add_address',
'del_address', 'add_personality', 'update_personality',
'del_personality', 'add_parameter', 'update_parameter',
'del_parameter', 'add_required_service', 'bind_feature',
'unbind_feature', 'validate_parameter', 'map_grn',
'unmap_grn', 'make', 'make_cluster'
]:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'alias_manager':
if action not in ['add_alias', 'del_alias', 'update_alias']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'webops':
if action not in [
'add_address', 'del_address', 'add_alias', 'del_alias',
'update_alias'
]:
self.raise_auth_error(principal, action, resource)
return True
def _check_aquilonhost(self, principal, dbuser, action, resource):
""" Return true if the incoming user is an aquilon host and this is
one of the few things that a host is allowed to change on its
own."""
if dbuser.name != 'aquilonhost':
return False
m = host_re.match(principal)
if not m:
return False
if resource.startswith("/host/%s/" % m.group(1)):
return True
return False
def check_network_environment(self, dbuser, dbnet_env):
"""More hacky authorization code.
This bit is a helper for restricting people from touching networks
that are maintained in other systems. That restriction can be
overridden for a configured list of roles (like aqd_admin) for
emergencies.
Something saner should be done here when we have better entitlements.
"""
if not dbnet_env.is_default:
# rely on standard authorization for all other environments
return
default = self.config.get("site", "default_network_environment")
conf_value = self.config.get("site", "change_default_netenv_roles")
allowed_roles = conf_value.strip().split()
if dbuser.role.name not in allowed_roles:
raise AuthorizationException("Only users with %s can modify "
"networks in the %s network "
"environment." %
(allowed_roles, default))
return
class AuthorizationBroker(object):
"""Handles any behind the scenes work in figuring out entitlements."""
# Borg
__shared_state = {}
def __init__(self):
self.__dict__ = self.__shared_state
self.config = Config()
def raise_auth_error(self, principal, action, resource):
auth_msg = self.config.get("broker", "authorization_error")
raise AuthorizationException("Unauthorized access attempt by %s to %s "
"on %s. %s" %
(principal, action, resource, auth_msg))
# FIXME: Hard coded check for now.
def check(self, principal, dbuser, action, resource):
if action.startswith('show') or action.startswith('search') \
or action.startswith('cat') or action == 'status' \
or action == 'dump_dns':
return True
if dbuser is None:
raise AuthorizationException(
"Unauthorized anonymous access attempt to %s on %s" %
(action, resource))
# Special-casing the aquilon hosts... this is a special user
# that provides a bucket for all host-generated activity.
if self._check_aquilonhost(principal, dbuser, resource):
return True
if dbuser.role.name == 'nobody':
self.raise_auth_error(principal, action, resource)
# Right now, anybody in a group can do anything they want, except...
if action in ['add_archetype', 'update_archetype', 'del_archetype',
'add_organization', 'del_organization',
'add_grn', 'del_grn', 'update_grn',
'add_vlan', 'del_vlan',
'rollback', 'update_city']:
if dbuser.role.name not in ['engineering', 'aqd_admin']:
raise AuthorizationException(
"Must have the engineering or aqd_admin role to %s." %
action)
if action in ['add_vendor', 'del_vendor',
'add_os', 'update_os', 'del_os',
'add_model', 'update_model', 'del_model']:
if dbuser.role.name not in ['engineering', 'aqd_admin',
'network_engineering']:
raise AuthorizationException(
"Must have the engineering, network_engineering or "
"aqd_admin role to %s." % action)
if action in ['permission']:
if dbuser.role.name not in ['aqd_admin', 'gatekeeper', 'secadmin']:
raise AuthorizationException(
"Must have the gatekeeper or aqd_admin role to %s." %
action)
if action in ['flush', 'refresh_windows_hosts']:
if dbuser.role.name not in ['aqd_admin']:
raise AuthorizationException(
"Must have the aqd_admin role to %s." % action)
if dbuser.role.name == 'winops':
if action not in ['add_host', 'add_host_prefix', 'add_windows_host',
'make_cluster', 'reconfigure', 'update_machine']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'winops_server':
# Only need add/update_cluster for hacluster VCenters
if action not in ['add_windows_host', 'del_windows_host',
'add_aurora_host',
'add_alias', 'update_alias', 'del_alias',
'add_machine',
'add_host', 'add_host_prefix',
'add_interface_hostname',
'add_interface_machine',
'add_interface_address',
'add_address', 'del_address',
'reconfigure',
'add_cluster', 'make_cluster', 'update_cluster',
'change_status', 'change_status_cluster',
'add_service_instance', 'map_service',
'bind_server', 'update_machine']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'mssb_unixops':
if action not in ['add_machine', 'del_machine',
'update_machine', 'update_machine_hostname',
'add_interface_hostname',
'add_interface_machine',
'add_interface_address',
'add_interface_chassis',
'update_interface_hostname',
'update_interface_machine',
'del_interface', 'del_interface_address',
'add_alias', 'update_alias', 'del_alias',
'add_address', 'del_address',
'add_host', 'add_host_prefix', 'del_host',
'add_windows_host', 'del_windows_host',
'add_manager', 'add_dynamic_range', 'add_disk',
'add_auxiliary',
'del_manager', 'del_auxiliary',
'del_disk', 'del_disk_disk', 'update_disk',
'add_filesystem', 'del_filesystem',
'add_rack', 'add_rack_room', 'add_chassis',
'del_rack', 'del_chassis',
'map_grn', 'unmap_grn',
'unmap_grn_clearall',
'change_status']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'spot_server':
if action not in ['add_machine', 'del_machine',
'add_machine_prefix',
'add_interface_hostname',
'add_interface_machine',
'add_interface_address',
'del_interface', 'del_interface_address',
'add_address', 'del_address',
'add_host', 'add_host_prefix', 'del_host',
'add_alias', 'del_alias',
'make', 'make_cluster',
'pxeswitch',
'change_status',
'add_room', 'add_rack', 'add_rack_room',
'add_disk', 'del_disk', 'del_disk_disk',
'update_disk']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'resource_pool':
if action not in ['add_address', 'del_address']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'secadmin':
if action not in ['permission']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'hpevelo':
if action not in ['reconfigure', 'pxeswitch', 'change_status',
'add_disk', 'del_disk', 'del_disk_disk',
'update_disk']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'location_admin':
if action not in ['add_building', 'del_building',
'add_city', 'del_city']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'telco_operations':
if action not in ['add_rack', 'add_network_device',
'add_switch',
'update_rack', 'update_network_device',
'update_switch',
'del_rack', 'del_network_device',
'del_switch',
'add_interface_network_device',
'add_interface_switch',
'del_interface_network_device',
'del_interface_switch',
'update_interface_network_device',
'update_interface_switch',
'add_interface_address_network_device',
'add_interface_address_switch',
'del_interface_address_network_device',
'del_interface_address_switch',
'add_alias', 'del_alias',
'refresh_network',
'update_router']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'network_engineering':
if action not in ['add_vendor', 'del_vendor',
'add_os', 'update_os', 'del_os',
'add_model', 'update_model', 'del_model',
'add_rack', 'add_network_device',
'add_switch',
'update_rack', 'update_network_device',
'update_switch',
'del_rack', 'del_network_device', 'del_switch',
'add_interface_network_device',
'add_interface_switch',
'del_interface_network_device',
'del_interface_switch',
'update_interface_network_device',
'update_interface_switch',
'add_interface_address_network_device',
'add_interface_address_switch',
'del_interface_address_network_device',
'del_interface_address_switch',
'add_alias', 'del_alias',
'refresh_network',
'update_router']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'edc':
if action not in ['add_rack', 'add_machine',
'add_host', 'add_host_prefix',
'add_interface_machine',
'add_interface_hostname',
'add_interface_address',
'add_alias', 'add_manager',
'add_service_address',
'del_service_address',
'update_rack', 'update_machine', 'update_alias',
'update_interface_hostname',
'update_interface_machine',
'del_rack', 'del_machine', 'del_host',
'del_interface', 'del_interface_address',
'del_alias', 'del_manager']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'maintech':
if action not in ['pxeswitch', 'pxeswitch_list',
'compile', 'compile_hostname', 'change_status',
'update_interface_hostname',
'update_interface_machine']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'template_admin':
if action not in ['add_sandbox', 'del_sandbox',
'manage', 'publish', 'reconfigure']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'laf':
if action not in ['reconfigure',
'add_reboot_intervention',
'compile_hostname', 'compile_personality',
'add_reboot_schedule',
'del_reboot_intervention',
'del_reboot_schedule',
'map_grn', 'unmap_grn',
'unmap_grn_clearall',
'add_hostlink', 'del_hostlink',
'add_personality', 'update_personality',
'del_personality',
'add_parameter', 'update_parameter',
'del_parameter',
'bind_feature', 'unbind_feature',
'map_service',
'add_alias', 'del_alias', 'update_alias']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'itsec':
if action not in ['pxeswitch', 'pxeswitch_list',
'compile', 'compile_hostname',
'change_status', 'make',
'reconfigure',
'reconfigure_list',
'reconfigure_hostlist',
'add_interface_machine',
'add_interface_hostname',
'add_interface_address',
'update_interface_hostname',
'update_interface_machine',
'del_interface', 'del_interface_address',
'add_disk', 'del_disk', 'del_disk_disk',
'update_disk',
'add_machine', 'del_machine',
'update_machine', 'update_machine_hostname',
'add_host', 'add_host_prefix', 'del_host',
'add_alias', 'add_manager',
'del_alias', 'del_manager']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'unixops_l2':
if action not in ['add_host', 'add_host_prefix', 'add_windows_host',
'del_host', 'del_windows_host',
'compile', 'compile_hostname',
'compile_cluster', 'compile_personality',
'reconfigure', 'change_status',
'reconfigure_list', 'reconfigure_hostlist',
'pxeswitch', 'pxeswitch_list',
'add_interface_chassis',
'add_interface_hostname',
'add_interface_machine',
'add_interface_address',
'del_interface',
'del_interface_address',
'add_reboot_intervention',
'add_reboot_schedule',
'del_reboot_intervention',
'del_reboot_schedule',
'update_interface_hostname',
'update_interface_machine',
'add_machine',
'del_machine',
'update_machine', 'update_machine_hostname',
'add_cluster', 'update_cluster',
'del_cluster',
'add_esx_cluster', 'update_esx_cluster',
'del_esx_cluster',
'bind_esx_cluster_hostname',
'rebind_esx_cluster_hostname',
'cluster', 'uncluster', 'change_status_cluster',
'add_manager', 'add_dynamic_range', 'add_disk',
'add_auxiliary',
'del_manager', 'del_auxiliary',
'del_disk', 'del_disk_disk', 'update_disk',
'add_service_instance',
'update_service_instance',
'del_service_instance',
'add_alias', 'del_alias',
'update_alias',
'add_filesystem',
'del_filesystem',
'poll_switch', 'poll_switch_switch',
'poll_network_device',
'poll_network_device_network_device',
'add_rack', 'add_rack_room', 'add_chassis',
'del_rack', 'del_chassis',
'add_bunker', 'del_bunker',
'add_address', 'del_address',
'add_personality', 'update_personality',
'del_personality',
'add_parameter', 'update_parameter',
'del_parameter', 'add_required_service',
'add_required_service_personality',
'bind_feature', 'unbind_feature',
'validate_parameter',
'add_hostlink', 'del_hostlink',
'add_static_route',
'del_static_route',
'manage_hostname', 'manage_list',
'manage_cluster',
'map_grn', 'unmap_grn',
'unmap_grn_clearall',
'make', 'make_cluster']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'alias_manager':
if action not in ['add_alias', 'del_alias', 'update_alias']:
self.raise_auth_error(principal, action, resource)
if dbuser.role.name == 'webops':
if action not in ['add_address', 'del_address',
'update_address',
'add_alias', 'del_alias', 'update_alias']:
self.raise_auth_error(principal, action, resource)
return True
def _check_aquilonhost(self, principal, dbuser, resource):
""" Return true if the incoming user is an aquilon host and this is
one of the few things that a host is allowed to change on its
own."""
if dbuser.name != 'aquilonhost':
return False
m = host_re.match(principal)
if not m:
return False
if resource.startswith("/host/%s/" % m.group(1)):
return True
return False
def check_network_environment(self, dbuser, dbnet_env):
"""More hacky authorization code.
This bit is a helper for restricting people from touching networks
that are maintained in other systems. That restriction can be
overridden for a configured list of roles (like aqd_admin) for
emergencies.
Something saner should be done here when we have better entitlements.
"""
if not dbnet_env.is_default:
# rely on standard authorization for all other environments
return
default = self.config.get("site", "default_network_environment")
conf_value = self.config.get("site", "change_default_netenv_roles")
allowed_roles = conf_value.strip().split()
if dbuser.role.name not in allowed_roles:
raise AuthorizationException("Only users with %s can modify "
"networks in the %s network "
"environment." %
(allowed_roles, default))
return
def __init__(self, dbsandbox, dbauthor):
self.dbsandbox = dbsandbox
self.dbauthor = dbauthor
config = Config()
templatesdir = config.get('broker', 'templatesdir')
self.path = os.path.join(templatesdir, dbauthor.name, dbsandbox.name)
%s
and use
%s
instead.""" % (os.environ["AQDCONF"], opts.config))
config = Config(configfile=opts.config)
if not config.has_section("unittest"):
config.add_section("unittest")
if not config.has_option("unittest", "srcdir"):
config.set("unittest", "srcdir", SRCDIR)
if opts.coverage:
config.set("unittest", "coverage", "True")
if opts.profile:
config.set("unittest", "profile", "True")
hostname = config.get("unittest", "hostname")
if hostname.find(".") < 0:
print >> sys.stderr, """
Some regression tests depend on the config value for hostname to be
fully qualified. Please set the config value manually since the default
on this system (%s) is a short name.
""" % hostname
sys.exit(1)
if opts.mirror:
# Copy the source directory and exec from it.
env = os.environ.copy()
if 'AQDCONF' in env:
env.pop('AQDCONF')
srcdir = config.get('unittest', 'srcdir')
mirrordir = config.get('unittest', 'mirrordir')
class AQBroker(object):
default_dir = os.path.dirname(__file__)
default_twistd = os.path.realpath(os.path.join(default_dir,
'..', '..',
'bin', 'twistd.py'))
default_configfile = os.path.realpath(os.path.join(default_dir,
'aqd.conf.scale'))
def __init__(self, twistd=None, configfile=None):
self.twistd = twistd or self.default_twistd
self.configfile = configfile or self.default_configfile
self.config = Config(configfile=self.configfile)
self.pidfile = os.path.join(self.config.get("broker", "rundir"),
"aqd.pid")
self.logfile = self.config.get("broker", "logfile")
self.coverage = os.path.join(self.config.get("broker", "logdir"),
"aqd.coverage")
def start(self, **kwargs):
"""Start a broker with the given config."""
# FIXME: Make coverage configurable.
args = [self.twistd, "--pidfile", self.pidfile,
"--logfile", self.logfile,
"aqd",
# "--coverage", self.coverage,
"--config", self.configfile]
p = Popen(args, stdout=1, stderr=2)
return p.wait()
def stop(self, **kwargs):
"""Attempt to stop a running broker."""
if os.path.exists(self.pidfile):
f = file(self.pidfile)
pid = f.readline()
f.close()
os.kill(int(pid), signal.SIGTERM)
def get_aqservice(self):
return self.config.get("broker", "service")
def initialize(self, **kwargs):
"""Clear out and set up the base directory and database.
Most of this was ripped straight from runtests.py.
"""
p = Popen(self.config.get("kerberos", "krb5_keytab"),
stdout=1, stderr=2)
if p.wait():
raise ProcessException(code=p.returncode)
for label in ["quattordir", "swrepdir", ]:
dir = self.config.get("broker", label)
if os.path.exists(dir):
continue
try:
os.makedirs(dir)
except OSError, e:
print >>sys.stderr, "Could not create %s: %s" % (dir, e)
dirs = [self.config.get("database", "dbdir")]
for label in ["domainsdir", "kingdir", "rundir", "profilesdir",
"plenarydir", "logdir"]:
dirs.append(self.config.get("broker", label))
for dir in dirs:
if os.path.exists(dir):
print "Removing %s" % dir
p = Popen(("/bin/rm", "-rf", dir), stdout=1, stderr=2)
if p.wait():
raise ProcessException(code=p.returncode)
try:
os.makedirs(dir)
except OSError, e:
print >>sys.stderr, "Could not create %s: %s" % (dir, e)
class Cloner(object):
@staticmethod
def get_parser():
msg = """Create a dev copy of prod data.
For a dev environment, just call the script on its own, either
with AQDCONF set or using the --configfile option. This will
connect to prod, exp data, and rsync templates.
To set up a QA environment, call the script with the --touser
option. This will exp the data and then imp it as the appropriate
user. The template files will be rsync'd into your directory.
Afterwards, re-run the script as the right id, passing in the
--finishfrom argument.
To test a database upgrade script, run with --schemaonly. This
will exp and imp as normal, but not imp any rows. Afterwards,
run build_db.py with the populate option. To use runtests.py,
disable the call to the AQDB tests that would rebuild the
database!
"""
parser = argparse.ArgumentParser(
description=msg,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--configfile',
help='Config file to use instead of AQDCONF')
parser.add_argument('--touser',
help='Specify a different target user/schema '
'for the database import')
parser.add_argument('--finishfrom',
help='Skip the database actions and do the '
'final rsync from this destination')
parser.add_argument('--schemaonly',
action='store_true',
help='Skip the rsync and do not imp any rows '
'into the database')
return parser
def __init__(self, args):
self.config = Config(configfile=args.configfile)
self.do_create = True
self.do_clear = True
self.do_restore = True
self.do_audit = True
self.do_schemaonly = False
self.do_rsync = True
self.warnings = []
self.source_db = 'NYPO_AQUILON_11'
self.source_exp = '/@%s' % self.source_db
self.target_dsn = self.config.get('database', 'dsn')
self.target_db = self.config.get('database', 'server')
self.target_imp = '/@%s' % self.target_db
if args.touser:
self.target_schema = args.touser.upper()
else:
self.target_schema = self.config.get('database', 'user').upper()
self.audit_schema = os.path.join(self.config.get('database',
'srcdir'), 'upgrade',
'1.7.6', 'add_transaction_tables.sql')
self.dbdir = self.config.get('database', 'dbdir')
self.dumpfile = os.path.join(self.dbdir, 'clone.dmp')
self.source_dir = 'nyaqd1:/var/quattor/'
self.target_dir = self.config.get('broker', 'quattordir')
self.rsync_filter = os.path.join(BINDIR, 'broker_sync.filter')
if args.touser:
self.do_audit = False
self.warnings.append("Sync'd data to: %s/" % self.target_dir)
if args.finishfrom:
self.do_create = False
self.do_clear = False
self.do_restore = False
self.do_audit = True
self.source_dir = args.finishfrom
if args.schemaonly:
self.do_create = True
self.do_clear = True
self.do_restore = True
self.do_schemaonly = True
self.do_rsync = False
if self.do_audit and not os.path.exists(self.audit_schema):
print >> sys.stderr, "Audit schema %s missing." % self.audit_schema
sys.exit(1)
if not self.target_dsn.startswith('oracle'):
print >> sys.stderr, 'Can only copy into an Oracle database.'
sys.exit(1)
if self.target_db == self.source_db or \
self.target_dsn.endswith(self.source_db):
print >>sys.stderr, \
'Check to make sure config does not point at prod.'
sys.exit(1)
def cloneprod(self):
self.create_dumpfile()
self.clear_target()
self.restore_dumpfile()
self.add_audit()
self.run_rsync()
if self.warnings:
print >> sys.stderr, "/n".join(self.warnings)
def create_dumpfile(self):
if not self.do_create:
return
if not os.path.exists(self.dbdir):
os.makedirs(self.dbdir)
if os.path.exists(self.dumpfile):
os.remove(self.dumpfile)
exp_args = [
'exp', self.source_exp,
'FILE=%s' % self.dumpfile, 'OWNER=cdb', 'CONSISTENT=y', 'DIRECT=y'
]
if self.do_schemaonly:
exp_args.append('ROWS=n')
p = Popen(exp_args, stdout=1, stderr=2)
p.communicate()
if p.returncode != 0:
print >> sys.stderr, "exp of %s failed!" % self.source_exp
sys.exit(p.returncode)
def clear_target(self):
if not self.do_clear:
return
nuke_target(self.target_imp, self.target_schema)
def restore_dumpfile(self):
if not self.do_restore:
return
p = Popen([
'imp', self.target_imp,
'FILE=%s' % self.dumpfile, 'IGNORE=y', 'SKIP_UNUSABLE_INDEXES=y',
'FROMUSER=CDB',
'TOUSER=%s' % self.target_schema
],
stdout=1,
stderr=2)
p.communicate()
if p.returncode != 0:
print >> sys.stderr, "imp to %s failed!" % self.target_imp
sys.exit(p.returncode)
def add_audit(self):
if not self.do_audit:
return
p = Popen(['sqlplus', self.target_imp,
'@%s' % self.audit_schema],
stdout=1,
stderr=2)
p.communicate()
if p.returncode != 0:
print >> sys.stderr, "sqlplus to %s failed!" % self.target_imp
sys.exit(p.returncode)
def run_rsync(self):
if not self.do_rsync:
return
if not os.path.exists(self.target_dir):
os.makedirs(self.target_dir)
p = Popen([
'rsync', '-avP', '--delete', '-e',
'ssh -q -o StrictHostKeyChecking=no -o ' +
'UserKnownHostsFile=/dev/null -o BatchMode=yes',
'--filter=merge %s' % self.rsync_filter, self.source_dir,
self.target_dir
],
stdout=1,
stderr=2)
p.communicate()
if p.returncode != 0:
print >> sys.stderr, "Rsync failed!"
sys.exit(p.returncode)
from sqlalchemy.exc import DBAPIError, DatabaseError as SaDBError
from sqlalchemy.schema import CreateIndex
from sqlalchemy.dialects.oracle.base import OracleDDLCompiler
import ms.modulecmd as modcmd
try:
config = Config()
except Exception, e:
print >> sys.stderr, 'failed to read configuration: %s' % e
sys.exit(os.EX_CONFIG)
assert config, 'No configuration in db_factory'
if config.has_option("database", "module"):
modcmd.load(config.get("database", "module"))
# Add support for Oracle-specific index extensions
@compiles(CreateIndex, 'oracle')
def visit_create_index(create, compiler, **kw):
index = create.element
compiler._verify_index_table(index)
preparer = compiler.preparer
text = "CREATE "
if index.unique:
text += "UNIQUE "
if index.kwargs.get("oracle_bitmap", False):
text += "BITMAP "
__table_args__ = (PrimaryKeyConstraint(xtn_id,
name,
value,
name="xtn_dtl_pk"),
Index('xtn_dtl_name_idx', name, oracle_compress=True),
Index('xtn_dtl_value_idx', value,
oracle_compress=True), {
'oracle_compress': 'OLTP'
})
Xtn.args = relationship(XtnDetail, lazy="joined", order_by=[XtnDetail.name])
if config.has_option('database', 'audit_schema'): # pragma: no cover
schema = config.get('database', 'audit_schema')
Xtn.__table__.schema = schema
XtnEnd.__table__.schema = schema
XtnDetail.__table__.schema = schema
def start_xtn(session,
xtn_id,
username,
command,
is_readonly,
details,
options_to_split=None):
""" Wrapper to log the start of a transaction (or running command).
Takes a dictionary with the transaction parameters. The keys are
class Cloner(object):
@staticmethod
def get_parser():
msg = """Create a dev copy of prod data.
For a dev environment, just call the script on its own, either
with AQDCONF set or using the --configfile option. This will
connect to prod, exp data, and rsync templates.
To set up a QA environment, call the script with the --touser
option. This will exp the data and then imp it as the appropriate
user. The template files will be rsync'd into your directory.
Afterwards, re-run the script as the right id, passing in the
--finishfrom argument.
To test a database upgrade script, run with --schemaonly. This
will exp and imp as normal, but not imp any rows. Afterwards,
run build_db.py with the populate option. To use runtests.py,
disable the call to the AQDB tests that would rebuild the
database!
"""
parser = argparse.ArgumentParser(
description=msg,
formatter_class=argparse.RawDescriptionHelpFormatter)
parser.add_argument('--configfile',
help='Config file to use instead of AQDCONF')
parser.add_argument('--touser',
help='Specify a different target user/schema '
'for the database import')
parser.add_argument('--finishfrom',
help='Skip the database actions and do the '
'final rsync from this destination')
parser.add_argument('--schemaonly', action='store_true',
help='Skip the rsync and do not imp any rows '
'into the database')
return parser
def __init__(self, args):
self.config = Config(configfile=args.configfile)
self.do_create = True
self.do_clear = True
self.do_restore = True
self.do_audit = True
self.do_schemaonly = False
self.do_rsync = True
self.warnings = []
self.source_db = 'NYPO_AQUILON_11'
self.source_exp = '/@%s' % self.source_db
self.target_dsn = self.config.get('database', 'dsn')
self.target_db = self.config.get('database', 'server')
self.target_imp = '/@%s' % self.target_db
if args.touser:
self.target_schema = args.touser.upper()
else:
self.target_schema = self.config.get('database', 'user').upper()
self.audit_schema = os.path.join(self.config.get('database', 'srcdir'),
'upgrade', '1.7.6',
'add_transaction_tables.sql')
self.dbdir = self.config.get('database', 'dbdir')
self.dumpfile = os.path.join(self.dbdir, 'clone.dmp')
self.source_dir = 'nyaqd1:/var/quattor/'
self.target_dir = self.config.get('broker', 'quattordir')
self.rsync_filter = os.path.join(BINDIR, 'broker_sync.filter')
if args.touser:
self.do_audit = False
self.warnings.append("Sync'd data to: %s/" % self.target_dir)
if args.finishfrom:
self.do_create = False
self.do_clear = False
self.do_restore = False
self.do_audit = True
self.source_dir = args.finishfrom
if args.schemaonly:
self.do_create = True
self.do_clear = True
self.do_restore = True
self.do_schemaonly = True
self.do_rsync = False
if self.do_audit and not os.path.exists(self.audit_schema):
print >>sys.stderr, "Audit schema %s missing." % self.audit_schema
sys.exit(1)
if not self.target_dsn.startswith('oracle'):
print >>sys.stderr, 'Can only copy into an Oracle database.'
sys.exit(1)
if self.target_db == self.source_db or \
self.target_dsn.endswith(self.source_db):
print >>sys.stderr, \
'Check to make sure config does not point at prod.'
sys.exit(1)
def cloneprod(self):
self.create_dumpfile()
self.clear_target()
self.restore_dumpfile()
self.add_audit()
self.run_rsync()
if self.warnings:
print >>sys.stderr, "/n".join(self.warnings)
def create_dumpfile(self):
if not self.do_create:
return
if not os.path.exists(self.dbdir):
os.makedirs(self.dbdir)
if os.path.exists(self.dumpfile):
os.remove(self.dumpfile)
exp_args = ['exp', self.source_exp, 'FILE=%s' % self.dumpfile,
'OWNER=cdb', 'CONSISTENT=y', 'DIRECT=y']
if self.do_schemaonly:
exp_args.append('ROWS=n')
p = Popen(exp_args, stdout=1, stderr=2)
p.communicate()
if p.returncode != 0:
print >>sys.stderr, "exp of %s failed!" % self.source_exp
sys.exit(p.returncode)
def clear_target(self):
if not self.do_clear:
return
nuke_target(self.target_imp, self.target_schema)
def restore_dumpfile(self):
if not self.do_restore:
return
p = Popen(['imp', self.target_imp, 'FILE=%s' % self.dumpfile,
'IGNORE=y', 'SKIP_UNUSABLE_INDEXES=y', 'FROMUSER=CDB',
'TOUSER=%s' % self.target_schema],
stdout=1, stderr=2)
p.communicate()
if p.returncode != 0:
print >>sys.stderr, "imp to %s failed!" % self.target_imp
sys.exit(p.returncode)
def add_audit(self):
if not self.do_audit:
return
p = Popen(['sqlplus', self.target_imp, '@%s' % self.audit_schema],
stdout=1, stderr=2)
p.communicate()
if p.returncode != 0:
print >>sys.stderr, "sqlplus to %s failed!" % self.target_imp
sys.exit(p.returncode)
def run_rsync(self):
if not self.do_rsync:
return
if not os.path.exists(self.target_dir):
os.makedirs(self.target_dir)
p = Popen(['rsync', '-avP', '--delete', '-e',
'ssh -q -o StrictHostKeyChecking=no -o ' +
'UserKnownHostsFile=/dev/null -o BatchMode=yes',
'--filter=merge %s' % self.rsync_filter,
self.source_dir, self.target_dir],
stdout=1, stderr=2)
p.communicate()
if p.returncode != 0:
print >>sys.stderr, "Rsync failed!"
sys.exit(p.returncode)
class TestBrokerCommand(unittest.TestCase):
def setUp(self):
self.config = Config()
self.net = DummyNetworks()
# Need to import protocol buffers after we have the config
# object all squared away and we can set the sys.path
# variable appropriately.
# It would be simpler just to change sys.path in runtests.py,
# but this allows for each test to be run individually (without
# the runtests.py wrapper).
protodir = self.config.get("protocols", "directory")
if protodir not in sys.path:
sys.path.append(protodir)
for m in ['aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2',
'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2',
'aqdparamdefinitions_pb2', 'aqdparameters_pb2']:
globals()[m] = __import__(m)
self.user = self.config.get("broker", "user")
self.sandboxdir = os.path.join(self.config.get("broker",
"templatesdir"),
self.user)
self.template_extension = self.config.get("panc", "template_extension")
# This method is cumbersome. Should probably develop something
# like unittest.conf.defaults.
if self.config.has_option("unittest", "scratchdir"):
self.scratchdir = self.config.get("unittest", "scratchdir")
if not os.path.exists(self.scratchdir):
os.makedirs(self.scratchdir)
if self.config.has_option("unittest", "aurora_with_node"):
self.aurora_with_node = self.config.get("unittest",
"aurora_with_node")
else:
self.aurora_with_node = "oyidb1622"
if self.config.has_option("unittest", "aurora_without_node"):
self.aurora_without_node = self.config.get("unittest",
"aurora_without_node")
else:
self.aurora_without_node = "pissp1"
self.gzip_profiles = self.config.getboolean("panc", "gzip_output")
self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml"
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
for name in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE,
DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR]:
path = os.path.join(dsdb_coverage_dir, name)
try:
os.remove(path)
except OSError:
pass
def tearDown(self):
pass
def template_name(self, *template, **args):
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "template_name() called without domain or sandbox")
return os.path.join(dir, *template) + self.template_extension
def plenary_name(self, *template):
dir = self.config.get("broker", "plenarydir")
return os.path.join(dir, *template) + self.template_extension
def find_template(self, *template, **args):
""" Figure out the extension of an existing template """
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "find_template() called without domain or sandbox")
base = os.path.join(dir, *template)
for extension in [".tpl", ".pan"]:
if os.path.exists(base + extension):
return base + extension
self.assert_(0, "template %s does not exist with any extension" % base)
def build_profile_name(self, *template, **args):
base = os.path.join(self.config.get("broker", "builddir"),
"domains", args.get("domain"),
"profiles", *template)
return base + self.template_extension
msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n')
def runcommand(self, command, auth=True, **kwargs):
aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py")
if auth:
port = self.config.get("broker", "kncport")
else:
port = self.config.get("broker", "openport")
if isinstance(command, list):
args = [str(cmd) for cmd in command]
else:
args = [command]
args.insert(0, sys.executable)
args.insert(1, aq)
if "--aqport" not in args:
args.append("--aqport")
args.append(port)
if auth:
args.append("--aqservice")
args.append(self.config.get("broker", "service"))
else:
args.append("--noauth")
if "env" in kwargs:
# Make sure that kerberos tickets are still present if the
# environment is being overridden...
env = {}
for (key, value) in kwargs["env"].items():
env[key] = value
for (key, value) in os.environ.items():
if key.find("KRB") == 0 and key not in env:
env[key] = value
if 'USER' not in env:
env['USER'] = os.environ.get('USER', '')
kwargs["env"] = env
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Strip any msversion dev warnings out of STDERR
err = self.msversion_dev_re.sub('', err)
# Lock messages are pretty common...
err = err.replace('Client status messages disabled, '
'retries exceeded.\n', '')
err = LOCK_RE.sub('', err)
return (p, out, err)
def successtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def statustest(self, command, **kwargs):
(out, err) = self.successtest(command, **kwargs)
self.assertEmptyOut(out, command)
return err
def failuretest(self, command, returncode, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, returncode,
"Non-%s return code %s for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n"
% (returncode, p.returncode, command, out, err))
return (out, err)
def assertEmptyStream(self, name, contents, command):
self.assertEqual(contents, "",
"%s for %s was not empty:\n@@@\n'%s'\n@@@\n"
% (name, command, contents))
def assertEmptyErr(self, contents, command):
self.assertEmptyStream("STDERR", contents, command)
def assertEmptyOut(self, contents, command):
self.assertEmptyStream("STDOUT", contents, command)
def commandtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEmptyErr(err, command)
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n"
% (command, out))
return out
def noouttest(self, command, **kwargs):
out = self.commandtest(command, **kwargs)
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n"
% (command, out))
def ignoreoutputtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return
# Right now, commands are not implemented consistently. When that is
# addressed, this unit test should be updated.
def notfoundtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
if p.returncode == 0:
self.assertEqual(err, "",
"STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
else:
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(err.find("Not Found") >= 0,
"STDERR for %s did not include Not Found:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
def badrequesttest(self, command, ignoreout=False, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.failUnless(err.find("Bad Request") >= 0,
"STDERR for %s did not include Bad Request:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
if not ignoreout and "--debug" not in command:
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs):
(p, out, err) = self.runcommand(command, auth=auth, **kwargs)
self.assertEqual(p.returncode, 4,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(err.find("Unauthorized:") >= 0,
"STDERR for %s did not include Unauthorized:"
"\n@@@\n'%s'\n@@@\n" %
(command, err))
if msgcheck:
self.searchoutput(err, r"Unauthorized (anonymous )?access attempt",
command)
return err
def internalerrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 5,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 5, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.assertEqual(err.find("Internal Server Error"), 0,
"STDERR for %s did not start with "
"Internal Server Error:\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
def unimplementederrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 5,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 5, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.assertEqual(err.find("Not Implemented"), 0,
"STDERR for %s did not start with "
"Not Implemented:\n@@@\n'%s'\n@@@\n" %
(command, err))
return err
# Test for conflicting or invalid aq client options.
def badoptiontest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(p.returncode, 2,
"Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 2, out, err))
self.assertEqual(out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def partialerrortest(self, command, **kwargs):
# Currently these two cases behave the same way - same exit code
# and behavior.
return self.badoptiontest(command, **kwargs)
def matchoutput(self, out, s, command):
self.assert_(out.find(s) >= 0,
"output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def matchclean(self, out, s, command):
self.assert_(out.find(s) < 0,
"output for %s includes '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def searchoutput(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failUnless(m,
"output for %s did not match '%s':\n@@@\n'%s'\n@@@\n"
% (command, r, out))
return m
def searchclean(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failIf(m,
"output for %s matches '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
def parse_proto_msg(self, listclass, attr, msg, expect=None):
protolist = listclass()
protolist.ParseFromString(msg)
received = len(getattr(protolist, attr))
if expect is None:
self.failUnless(received > 0,
"No %s listed in %s protobuf message\n" %
(attr, listclass))
else:
self.failUnlessEqual(received, expect,
"%d %s expected, got %d\n" %
(expect, attr, received))
return protolist
def parse_netlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdnetworks_pb2.NetworkList,
'networks',
msg, expect)
def parse_hostlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.HostList,
'hosts',
msg, expect)
def parse_clusters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ClusterList,
'clusters',
msg, expect)
def parse_location_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdlocations_pb2.LocationList,
'locations',
msg, expect)
def parse_dns_domainlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList,
'dns_domains',
msg, expect)
def parse_service_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceList,
'services',
msg, expect)
def parse_servicemap_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceMapList,
'servicemaps',
msg, expect)
def parse_personality_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.PersonalityList,
'personalities',
msg, expect)
def parse_os_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList,
'operating_systems',
msg, expect)
def parse_audit_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdaudit_pb2.TransactionList,
'transactions', msg, expect)
def parse_resourcelist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ResourceList,
'resources',
msg, expect)
def parse_paramdefinition_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparamdefinitions_pb2.ParamDefinitionList,
'param_definitions', msg, expect)
def parse_parameters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparameters_pb2.ParameterList,
'parameters', msg, expect)
def gitenv(self, env=None):
"""Configure a known sanitised environment"""
git_path = self.config.get("broker", "git_path")
# The "publish" test abuses gitenv(), and it needs the Python interpreter
# in the path, because it runs the template unit tests which in turn
# call the aq command
python_path = os.path.dirname(sys.executable)
newenv = {}
newenv["USER"] = os.environ.get('USER', '')
if env:
for (key, value) in env.iteritems():
newenv[key] = value
if "PATH" in newenv:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, newenv["PATH"])
else:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path, '/bin:/usr/bin')
return newenv
def gitcommand_raw(self, command, **kwargs):
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, "git")
env = self.gitenv(kwargs.pop("env", None))
p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
return p
def gitcommand(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def gitcommand_expectfailure(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.failIfEqual(p.returncode, 0,
"Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def check_git_merge_health(self, repo):
command = "merge HEAD"
out = self.gitcommand(command.split(" "), cwd=repo)
return
def grepcommand(self, command, **kwargs):
if self.config.has_option("unittest", "grep"):
grep = self.config.get("unittest", "grep")
else:
grep = "/bin/grep"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, grep)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
if p.returncode == 1:
return []
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
def findcommand(self, command, **kwargs):
if self.config.has_option("unittest", "find"):
find = self.config.get("unittest", "find")
else:
find = "/usr/bin/find"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, find)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
def writescratch(self, filename, contents):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'w') as f:
f.write(contents)
return scratchfile
def readscratch(self, filename):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'r') as f:
contents = f.read()
return contents
def dsdb_expect(self, command, fail=False, errstr=""):
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
if fail:
filename = DSDB_EXPECT_FAILURE_FILE
else:
filename = DSDB_EXPECT_SUCCESS_FILE
expected_name = os.path.join(dsdb_coverage_dir, filename)
with open(expected_name, "a") as fp:
if isinstance(command, list):
fp.write(" ".join([str(cmd) for cmd in command]))
else:
fp.write(str(command))
fp.write("\n")
if fail and errstr:
errfile = DSDB_EXPECT_FAILURE_ERROR
expected_name = os.path.join(dsdb_coverage_dir, errfile)
with open(expected_name, "a") as fp:
fp.write(errstr)
fp.write("\n")
def dsdb_expect_add(self, hostname, ip, interface=None, mac=None,
primary=None, comments=None, fail=False):
command = ["add_host", "-host_name", hostname,
"-ip_address", str(ip), "-status", "aq"]
if interface:
command.extend(["-interface_name",
str(interface).replace('/', '_')])
if mac:
command.extend(["-ethernet_address", str(mac)])
if primary:
command.extend(["-primary_host_name", primary])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_delete(self, ip, fail=False):
self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail)
def dsdb_expect_update(self, fqdn, iface=None, ip=None, mac=None,
comments=None, fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if iface:
command.extend(["-interface_name", iface])
if ip:
command.extend(["-ip_address", str(ip)])
if mac:
command.extend(["-ethernet_address", str(mac)])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_rename(self, fqdn, new_fqdn=None, iface=None,
new_iface=None, fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if new_fqdn:
command.extend(["-primary_host_name", new_fqdn])
if iface:
command.extend(["-interface_name", iface])
if new_iface:
command.extend(["-new_interface_name", new_iface])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_add_campus(self, campus, comments=None, fail=False,
errstr=""):
command = ["add_campus_aq", "-campus_name", campus]
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus(self, campus, fail=False, errstr=""):
command = ["delete_campus_aq", "-campus", campus]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_add_campus_building(self, campus, building, fail=False,
errstr=""):
command = ["add_campus_building_aq", "-campus_name", campus,
"-building_name", building]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus_building(self, campus, building, fail=False,
errstr=""):
command = ["delete_campus_building_aq", "-campus_name", campus,
"-building_name", building]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_verify(self, empty=False):
dsdb_coverage_dir = os.path.join(self.config.get("unittest", "scratchdir"),
"dsdb_coverage")
fail_expected_name = os.path.join(dsdb_coverage_dir,
DSDB_EXPECT_FAILURE_FILE)
issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE)
expected = {}
for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]:
expected_name = os.path.join(dsdb_coverage_dir, filename)
try:
with open(expected_name, "r") as fp:
for line in fp:
expected[line.rstrip("\n")] = True
except IOError:
pass
# This is likely a logic error in the test
if not expected and not empty:
self.fail("dsdb_verify() called when no DSDB commands were "
"expected?!?")
issued = {}
try:
with open(issued_name, "r") as fp:
for line in fp:
issued[line.rstrip("\n")] = True
except IOError:
pass
errors = []
for cmd, dummy in expected.items():
if cmd not in issued:
errors.append("'%s'" % cmd)
# Unexpected DSDB commands are caught by the fake_dsdb script
if errors:
self.fail("The following expected DSDB commands were not called:"
"\n@@@\n%s\n@@@\n" % "\n".join(errors))
def verify_buildfiles(self, domain, object,
want_exist=True, command='manage'):
qdir = self.config.get('broker', 'quattordir')
domaindir = os.path.join(qdir, 'build', 'xml', domain)
xmlfile = os.path.join(domaindir, object + self.profile_suffix)
depfile = os.path.join(domaindir, object + '.dep')
builddir = self.config.get('broker', 'builddir')
profile = os.path.join(builddir, 'domains', domain, 'profiles',
object + self.template_extension)
for f in [xmlfile, depfile, profile]:
if want_exist:
self.failUnless(os.path.exists(f),
"Expecting %s to exist before running %s." %
(f, command))
else:
self.failIf(os.path.exists(f),
"Not expecting %s to exist after running %s." %
(f, command))
def demote_current_user(self, role="nobody"):
principal = self.config.get('unittest', 'principal')
command = ["permission", "--role", role, "--principal", principal]
self.noouttest(command)
def promote_current_user(self):
srcdir = self.config.get("broker", "srcdir")
add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py")
env = os.environ.copy()
env['AQDCONF'] = self.config.baseconfig
p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env)
(out, err) = p.communicate()
self.assertEqual(p.returncode, 0,
"Failed to restore admin privs '%s', '%s'." %
(out, err))
def __init__(self, dbsandbox, dbauthor):
self.dbsandbox = dbsandbox
self.dbauthor = dbauthor
config = Config()
templatesdir = config.get('broker', 'templatesdir')
self.path = os.path.join(templatesdir, dbauthor.name, dbsandbox.name)
class BrokerCommand(object):
""" The basis for each command module under commands.
Several class-level lists and flags are defined here that can be
overridden on a per-command basis. Some can only be overridden
in __init__, though, so check the docstrings.
"""
required_parameters = []
""" This will generally be overridden in the command class.
It could theoretically be parsed out of input.xml, but that is
tricky in some cases and possibly error-prone.
"""
optional_parameters = []
""" Optional parameters are filled in automatically from input.xml.
This may contain entries that are also in the required_parameters list.
If so, the required entry "wins".
"""
requires_azcheck = True
""" Opt out of authorization checks by setting this flag to False."""
requires_transaction = True
""" This sets up a session and cleans up when finished.
Currently, setting azcheck to True will force this value to be True,
because the azcheck requires a session that will need to be cleaned
up.
"""
requires_format = False
""" Run command results through the formatter.
It is automatically set to True for all cat, search, and show
commands, but could be reversed back to False by overriding __init__
for the command.
"""
requires_readonly = False
""" Require read only isolation level for the render session.
It is automatically set to True for all search and show commands,
but could be reversed back to False by overriding __init__ for the
command.
"""
# Override to indicate whether the command will generally take a
# lock during execution.
#
# Any command with this flag set to True will use the separate
# NLSession thread pool and should not have to wait on commands
# that are potentially all blocking on the same lock.
#
# If set to None (the default), the is_lock_free property will
# examine the command's module to try to determine if a lock
# may be required and then cache the value.
_is_lock_free = None
# Run the render method on a separate thread. This will be forced
# to True if requires_azcheck or requires_transaction.
defer_to_thread = True
def __init__(self):
""" Provides some convenient variables for commands.
Also sets requires_* parameters for some sets of commands.
All of the command objects are singletons (or Borg).
"""
self.dbf = DbFactory()
self.config = Config()
self.az = AuthorizationBroker()
self.formatter = ResponseFormatter()
self.catalog = StatusCatalog()
# Force the instance to have local copies of the class defaults...
# This allows resources.py to modify instances without worrying
# about inheritance issues (classes sharing required or optional
# parameters).
self.required_parameters = self.required_parameters[:]
self.optional_parameters = self.optional_parameters[:]
# Parameter checks are filled in automatically based on input.xml. This
# lets us do some rudimentary checks before the actual command is
# invoked.
self.parameter_checks = {}
# The parameter types are filled in automatically based on input.xml.
self.parameter_types = {}
# This is the pivot of the above, filled in at the same time. It is a
# dictionary of type names to lists of parameters.
self.parameters_by_type = {}
self.action = self.__module__
package_prefix = "aquilon.worker.commands."
if self.action.startswith(package_prefix):
self.action = self.action[len(package_prefix):]
# self.command is set correctly in resources.py after parsing input.xml
self.command = self.action
# The readonly and format flags are done here for convenience
# and simplicity. They could be overridden by the __init__
# method of any show/search/cat commands that do not want these
# defaults. Some 'one-off' commands (like ping and status)
# just set the variables themselves.
if self.action.startswith("show") or self.action.startswith("search"):
self.requires_readonly = True
self.requires_format = True
if self.action.startswith("cat"):
self.requires_format = True
self.requires_readonly = True
self._is_lock_free = True
if not self.requires_readonly \
and self.config.get('broker', 'mode') == 'readonly':
self.badmode = 'readonly'
else:
self.badmode = False
self._update_render(self.render)
if not self.defer_to_thread:
if self.requires_azcheck or self.requires_transaction:
self.defer_to_thread = True
log.msg("Forcing defer_to_thread to True because of "
"required authorization or transaction for %s" %
self.command)
# Not sure how to handle formatting with deferred...
self.requires_format = False
#free = "True " if self.is_lock_free else "False"
#log.msg("is_lock_free = %s [%s]" % (free, self.command))
def audit_result(self, session, key, value, **arguments):
# We need a place to store the result somewhere until we can finish the
# audit record. Use the request object for now.
request = arguments["request"]
if not hasattr(request, "_audit_result"):
request._audit_result = []
request._audit_result.append((key, value))
def render(self, **arguments): # pragma: no cover
""" Implement this method to create a functional broker command.
The base __init__ method wraps all implementations using
_update_render() to enforce the class requires_* flags.
"""
if self.__class__.__module__ == 'aquilon.worker.broker':
# Default class... no useful command info to repeat back...
raise UnimplementedError("Command has not been implemented.")
raise UnimplementedError("%s has not been implemented" %
self.__class__.__module__)
def _update_render(self, command):
""" Wrap the render method using the requires_* attributes.
An alternate implementation would be to just have a
wrap_rendor() method or similar that got called instead
of rendor().
"""
def updated_render(self, *args, **kwargs):
principal = kwargs["user"]
request = kwargs["request"]
logger = kwargs["logger"]
raising_exception = None
rollback_failed = False
try:
if self.requires_transaction or self.requires_azcheck:
# Set up a session...
if not "session" in kwargs:
if self.is_lock_free:
kwargs["session"] = self.dbf.NLSession()
else:
kwargs["session"] = self.dbf.Session()
session = kwargs["session"]
if session.bind.dialect.name == "oracle":
# Make the name of the command and the request ID
# available in v$session. Trying to set a value longer
# than the allowed length will generate ORA-24960, so
# do an explicit truncation.
conn = session.connection()
dbapi_con = conn.connection.connection
dbapi_con.action = str(self.action)[:32]
# TODO: we should include the command number as well,
# since that is easier to find in the logs
dbapi_con.clientinfo = str(kwargs["requestid"])[:64]
# This does a COMMIT, which in turn invalidates the session.
# We should therefore avoid looking up anything in the DB
# before this point which might be used later.
self._record_xtn(session, logger.get_status())
dbuser = get_or_create_user_principal(session, principal,
commitoncreate=True)
kwargs["dbuser"] = dbuser
if self.requires_azcheck:
self.az.check(principal=principal, dbuser=dbuser,
action=self.action,
resource=request.path)
if self.requires_readonly:
self._set_readonly(session)
# begin() is only required if session transactional=False
#session.begin()
if self.badmode: # pragma: no cover
raise UnimplementedError("Command %s not available on "
"a %s broker." %
(self.command, self.badmode))
for key in kwargs.keys():
if key in self.parameter_checks:
kwargs[key] = self.parameter_checks[key]("--" + key,
kwargs[key])
# Command is an instance method already having self...
retval = command(*args, **kwargs)
if self.requires_format:
style = kwargs.get("style", None)
retval = self.formatter.format(style, retval, request)
if "session" in kwargs:
session.commit()
return retval
except Exception, e:
raising_exception = e
# Need to close after the rollback, or the next time session
# is accessed it tries to commit the transaction... (?)
if "session" in kwargs:
try:
session.rollback()
except: # pragma: no cover
rollback_failed = True
raise
session.close()
raise
finally:
import os
import logging
from aquilon.exceptions_ import InternalError, IncompleteError
from aquilon.config import Config
from aquilon.worker.locks import lock_queue, CompileKey
from aquilon.worker.processes import write_file, read_file, remove_file
from aquilon.worker.templates.panutils import pan_assign, pan_variable
from mako.lookup import TemplateLookup
from aquilon.worker.formats.formatters import ObjectFormatter
LOGGER = logging.getLogger(__name__)
_config = Config()
TEMPLATE_EXTENSION = _config.get("panc", "template_extension")
class Plenary(object):
template_type = None
""" Specifies the PAN template type to generate """
handlers = {}
""" The handlers dictionary should have an entry for every subclass.
Typically this will be defined immediately after defining the
subclass.
"""
def __init__(self, dbobj=None, logger=LOGGER):
self.config = Config()
class Plenary(object):
template_type = None
""" Specifies the PAN template type to generate """
handlers = {}
""" The handlers dictionary should have an entry for every subclass.
Typically this will be defined immediately after defining the
subclass.
"""
def __init__(self, dbobj=None, logger=LOGGER):
self.config = Config()
self.dbobj = dbobj
self.logger = logger
if self.template_type is None:
raise InternalError("Plenary class %s did not set the template "
"type" % self.__class__.__name__)
# Object templates live under the branch-specific build directory.
# Everything else lives under the common plenary directory.
if self.template_type == "object":
if not dbobj or not hasattr(dbobj, "branch"):
raise InternalError("Plenaries meant to be compiled need a DB "
"object that has a branch; got: %r" %
dbobj)
self.dir = "%s/domains/%s/profiles" % (self.config.get(
"broker", "builddir"), dbobj.branch.name)
else:
self.dir = self.config.get("broker", "plenarydir")
self.loadpath = None
self.plenary_template = None
self.plenary_core = None
self.new_content = None
# The following attributes are for stash/restore_stash
self.old_content = None
self.old_mtime = None
self.stashed = False
self.removed = False
self.changed = False
def __hash__(self):
"""Since equality is based on dbobj, just hash on it."""
return hash(self.dbobj)
def __eq__(self, other):
"""Plenary objects are equal if they describe the same object.
Technically this should probably also check that the class
matches. There are some odd cases when the plenary stores
extra information, currently ignored.
"""
if self.dbobj is None or other.dbobj is None:
return False
return self.dbobj == other.dbobj
def __str__(self):
"""For debug output."""
return "Plenary(%s)" % self.dbobj
@property
def plenary_directory(self):
""" Directory where the plenary template lives """
if self.loadpath and self.template_type != "object":
return "%s/%s/%s" % (self.dir, self.loadpath, self.plenary_core)
else:
return "%s/%s" % (self.dir, self.plenary_core)
@property
def plenary_file(self):
""" Full absolute path name of the plenary template """
return "%s/%s%s" % (self.plenary_directory, self.plenary_template,
TEMPLATE_EXTENSION)
@property
def plenary_template_name(self):
""" Name of the template as used by PAN, relative to the load path """
if self.plenary_core:
return "%s/%s" % (self.plenary_core, self.plenary_template)
else:
return self.plenary_template
def body(self, lines):
"""
The text of the template. By default, do nothing. A derived class can
override this to describe their own content.
They should do this by appending strings (each string
referring to a separate line of text in the template) into the
array. The array will already contain the appropriate header line for the
template.
"""
pass
def will_change(self):
self.stash()
if not self.new_content:
self.new_content = self._generate_content()
return self.old_content != self.new_content
def get_write_key(self):
if self.will_change():
return self.get_key()
return None
def get_remove_key(self):
"""Return the relevant key.
In many cases it will be more efficient just to create a full
compile lock and bypass this method.
"""
return self.get_key()
def get_key(self):
"""Base implementation assumes a full compile lock."""
return CompileKey(logger=self.logger)
def _generate_content(self):
"""Not meant to be overridden or called directly."""
lines = []
type = self.template_type
if type is not None and type is not "":
type = type + " "
lines.append("%stemplate %s;" % (type, self.plenary_template_name))
lines.append("")
if self.template_type == "object":
if self.loadpath:
pan_variable(lines, "LOADPATH", [self.loadpath])
lines.append("")
pan_assign(lines, "/metadata/template/branch/name",
self.dbobj.branch.name)
pan_assign(lines, "/metadata/template/branch/type",
self.dbobj.branch.branch_type)
if self.dbobj.branch.branch_type == 'sandbox':
pan_assign(lines, "/metadata/template/branch/author",
self.dbobj.sandbox_author.name)
lines.append("")
self.body(lines)
return "\n".join(lines) + "\n"
def write(self, locked=False, content=None):
"""Write out the template.
If the content is unchanged, then the file will not be modified
(preserving the mtime).
Returns the number of files that were written.
If locked is True then it is assumed that error handling happens
higher in the call stack.
"""
if self.template_type == "object" and \
hasattr(self.dbobj, "personality") and \
self.dbobj.personality and \
not self.dbobj.personality.archetype.is_compileable:
return 0
if content is None:
if not self.new_content:
self.new_content = self._generate_content()
content = self.new_content
self.stash()
if self.old_content == content and \
not self.removed and not self.changed:
# optimise out the write (leaving the mtime good for ant)
# if nothing is actually changed
return 0
key = None
try:
if not locked:
key = self.get_write_key()
lock_queue.acquire(key)
if not os.path.exists(self.plenary_directory):
os.makedirs(self.plenary_directory)
write_file(self.plenary_file, content, logger=self.logger)
self.removed = False
if self.old_content != content:
self.changed = True
except Exception, e:
if not locked:
self.restore_stash()
raise e
finally:
class TestBrokerCommand(unittest.TestCase):
def setUp(self):
self.config = Config()
self.net = DummyNetworks()
# Need to import protocol buffers after we have the config
# object all squared away and we can set the sys.path
# variable appropriately.
# It would be simpler just to change sys.path in runtests.py,
# but this allows for each test to be run individually (without
# the runtests.py wrapper).
protodir = self.config.get("protocols", "directory")
if protodir not in sys.path:
sys.path.append(protodir)
for m in [
'aqdsystems_pb2', 'aqdnetworks_pb2', 'aqdservices_pb2',
'aqddnsdomains_pb2', 'aqdlocations_pb2', 'aqdaudit_pb2',
'aqdparamdefinitions_pb2', 'aqdparameters_pb2'
]:
globals()[m] = __import__(m)
self.user = self.config.get("broker", "user")
self.sandboxdir = os.path.join(
self.config.get("broker", "templatesdir"), self.user)
self.template_extension = self.config.get("panc", "template_extension")
# This method is cumbersome. Should probably develop something
# like unittest.conf.defaults.
if self.config.has_option("unittest", "scratchdir"):
self.scratchdir = self.config.get("unittest", "scratchdir")
if not os.path.exists(self.scratchdir):
os.makedirs(self.scratchdir)
if self.config.has_option("unittest", "aurora_with_node"):
self.aurora_with_node = self.config.get("unittest",
"aurora_with_node")
else:
self.aurora_with_node = "oyidb1622"
if self.config.has_option("unittest", "aurora_without_node"):
self.aurora_without_node = self.config.get("unittest",
"aurora_without_node")
else:
self.aurora_without_node = "pissp1"
self.gzip_profiles = self.config.getboolean("panc", "gzip_output")
self.profile_suffix = ".xml.gz" if self.gzip_profiles else ".xml"
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
for name in [
DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE,
DSDB_ISSUED_CMDS_FILE, DSDB_EXPECT_FAILURE_ERROR
]:
path = os.path.join(dsdb_coverage_dir, name)
try:
os.remove(path)
except OSError:
pass
def tearDown(self):
pass
def template_name(self, *template, **args):
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "template_name() called without domain or sandbox")
return os.path.join(dir, *template) + self.template_extension
def plenary_name(self, *template):
dir = self.config.get("broker", "plenarydir")
return os.path.join(dir, *template) + self.template_extension
def find_template(self, *template, **args):
""" Figure out the extension of an existing template """
if args.get("sandbox", None):
dir = os.path.join(self.sandboxdir, args.get("sandbox"))
elif args.get("domain", None):
dir = os.path.join(self.config.get("broker", "domainsdir"),
args.get("domain"))
else:
self.assert_(0, "find_template() called without domain or sandbox")
base = os.path.join(dir, *template)
for extension in [".tpl", ".pan"]:
if os.path.exists(base + extension):
return base + extension
self.assert_(0, "template %s does not exist with any extension" % base)
def build_profile_name(self, *template, **args):
base = os.path.join(self.config.get("broker", "builddir"), "domains",
args.get("domain"), "profiles", *template)
return base + self.template_extension
msversion_dev_re = re.compile('WARNING:msversion:Loading \S* from dev\n')
def runcommand(self, command, auth=True, **kwargs):
aq = os.path.join(self.config.get("broker", "srcdir"), "bin", "aq.py")
if auth:
port = self.config.get("broker", "kncport")
else:
port = self.config.get("broker", "openport")
if isinstance(command, list):
args = [str(cmd) for cmd in command]
else:
args = [command]
args.insert(0, sys.executable)
args.insert(1, aq)
if "--aqport" not in args:
args.append("--aqport")
args.append(port)
if auth:
args.append("--aqservice")
args.append(self.config.get("broker", "service"))
else:
args.append("--noauth")
if "env" in kwargs:
# Make sure that kerberos tickets are still present if the
# environment is being overridden...
env = {}
for (key, value) in kwargs["env"].items():
env[key] = value
for (key, value) in os.environ.items():
if key.find("KRB") == 0 and key not in env:
env[key] = value
if 'USER' not in env:
env['USER'] = os.environ.get('USER', '')
kwargs["env"] = env
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Strip any msversion dev warnings out of STDERR
err = self.msversion_dev_re.sub('', err)
# Lock messages are pretty common...
err = err.replace(
'Client status messages disabled, '
'retries exceeded.\n', '')
err = LOCK_RE.sub('', err)
return (p, out, err)
def successtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 0, "Non-zero return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n" % (command, out, err))
return (out, err)
def statustest(self, command, **kwargs):
(out, err) = self.successtest(command, **kwargs)
self.assertEmptyOut(out, command)
return err
def failuretest(self, command, returncode, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, returncode, "Non-%s return code %s for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\n"
"STDERR:\n@@@\n'%s'\n@@@\n" %
(returncode, p.returncode, command, out, err))
return (out, err)
def assertEmptyStream(self, name, contents, command):
self.assertEqual(
contents, "", "%s for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(name, command, contents))
def assertEmptyErr(self, contents, command):
self.assertEmptyStream("STDERR", contents, command)
def assertEmptyOut(self, contents, command):
self.assertEmptyStream("STDOUT", contents, command)
def commandtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEmptyErr(err, command)
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\n" %
(command, out))
return out
def noouttest(self, command, **kwargs):
out = self.commandtest(command, **kwargs)
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
def ignoreoutputtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return
# Right now, commands are not implemented consistently. When that is
# addressed, this unit test should be updated.
def notfoundtest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
if p.returncode == 0:
self.assertEqual(
err, "", "STDERR for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, err))
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
else:
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" %
(command, p.returncode, 4, out, err))
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
self.failUnless(
err.find("Not Found") >= 0,
"STDERR for %s did not include Not Found:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
return err
def badrequesttest(self, command, ignoreout=False, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err))
self.failUnless(
err.find("Bad Request") >= 0,
"STDERR for %s did not include Bad Request:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
if not ignoreout and "--debug" not in command:
self.assertEqual(
out, "", "STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" %
(command, out))
return err
def unauthorizedtest(self, command, auth=False, msgcheck=True, **kwargs):
(p, out, err) = self.runcommand(command, auth=auth, **kwargs)
self.assertEqual(
p.returncode, 4, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 4, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.failUnless(
err.find("Unauthorized:") >= 0,
"STDERR for %s did not include Unauthorized:"
"\n@@@\n'%s'\n@@@\n" % (command, err))
if msgcheck:
self.searchoutput(err, r"Unauthorized (anonymous )?access attempt",
command)
return err
def internalerrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 5, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.assertEqual(
err.find("Internal Server Error"), 0,
"STDERR for %s did not start with "
"Internal Server Error:\n@@@\n'%s'\n@@@\n" % (command, err))
return err
def unimplementederrortest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 5, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 5, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
self.assertEqual(
err.find("Not Implemented"), 0, "STDERR for %s did not start with "
"Not Implemented:\n@@@\n'%s'\n@@@\n" % (command, err))
return err
# Test for conflicting or invalid aq client options.
def badoptiontest(self, command, **kwargs):
(p, out, err) = self.runcommand(command, **kwargs)
self.assertEqual(
p.returncode, 2, "Return code for %s was %d instead of %d"
"\nSTDOUT:\n@@@\n'%s'\n@@@"
"\nSTDERR:\n@@@\n'%s'\n@@@" % (command, p.returncode, 2, out, err))
self.assertEqual(
out, "",
"STDOUT for %s was not empty:\n@@@\n'%s'\n@@@\n" % (command, out))
return err
def partialerrortest(self, command, **kwargs):
# Currently these two cases behave the same way - same exit code
# and behavior.
return self.badoptiontest(command, **kwargs)
def matchoutput(self, out, s, command):
self.assert_(
out.find(s) >= 0,
"output for %s did not include '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def matchclean(self, out, s, command):
self.assert_(
out.find(s) < 0, "output for %s includes '%s':\n@@@\n'%s'\n@@@\n" %
(command, s, out))
def searchoutput(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failUnless(
m, "output for %s did not match '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
return m
def searchclean(self, out, r, command):
if isinstance(r, str):
m = re.search(r, out, re.MULTILINE)
else:
m = re.search(r, out)
self.failIf(
m, "output for %s matches '%s':\n@@@\n'%s'\n@@@\n" %
(command, r, out))
def parse_proto_msg(self, listclass, attr, msg, expect=None):
protolist = listclass()
protolist.ParseFromString(msg)
received = len(getattr(protolist, attr))
if expect is None:
self.failUnless(
received > 0,
"No %s listed in %s protobuf message\n" % (attr, listclass))
else:
self.failUnlessEqual(
received, expect,
"%d %s expected, got %d\n" % (expect, attr, received))
return protolist
def parse_netlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdnetworks_pb2.NetworkList, 'networks',
msg, expect)
def parse_hostlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.HostList, 'hosts', msg,
expect)
def parse_clusters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ClusterList, 'clusters',
msg, expect)
def parse_location_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdlocations_pb2.LocationList, 'locations',
msg, expect)
def parse_dns_domainlist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqddnsdomains_pb2.DNSDomainList,
'dns_domains', msg, expect)
def parse_service_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceList, 'services',
msg, expect)
def parse_servicemap_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdservices_pb2.ServiceMapList,
'servicemaps', msg, expect)
def parse_personality_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.PersonalityList,
'personalities', msg, expect)
def parse_os_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.OperatingSystemList,
'operating_systems', msg, expect)
def parse_audit_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdaudit_pb2.TransactionList,
'transactions', msg, expect)
def parse_resourcelist_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdsystems_pb2.ResourceList, 'resources',
msg, expect)
def parse_paramdefinition_msg(self, msg, expect=None):
return self.parse_proto_msg(
aqdparamdefinitions_pb2.ParamDefinitionList, 'param_definitions',
msg, expect)
def parse_parameters_msg(self, msg, expect=None):
return self.parse_proto_msg(aqdparameters_pb2.ParameterList,
'parameters', msg, expect)
def gitenv(self, env=None):
"""Configure a known sanitised environment"""
git_path = self.config.get("broker", "git_path")
# The "publish" test abuses gitenv(), and it needs the Python interpreter
# in the path, because it runs the template unit tests which in turn
# call the aq command
python_path = os.path.dirname(sys.executable)
newenv = {}
newenv["USER"] = os.environ.get('USER', '')
if env:
for (key, value) in env.iteritems():
newenv[key] = value
if "PATH" in newenv:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path,
newenv["PATH"])
else:
newenv["PATH"] = "%s:%s:%s" % (git_path, python_path,
'/bin:/usr/bin')
return newenv
def gitcommand_raw(self, command, **kwargs):
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, "git")
env = self.gitenv(kwargs.pop("env", None))
p = Popen(args, stdout=PIPE, stderr=PIPE, env=env, **kwargs)
return p
def gitcommand(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0,
"Non-zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def gitcommand_expectfailure(self, command, **kwargs):
p = self.gitcommand_raw(command, **kwargs)
# Ignore out/err unless we get a non-zero return code, then log it.
(out, err) = p.communicate()
self.failIfEqual(
p.returncode, 0,
"Zero return code for %s, STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n"
% (command, out, err))
return (out, err)
def check_git_merge_health(self, repo):
command = "merge HEAD"
out = self.gitcommand(command.split(" "), cwd=repo)
return
def grepcommand(self, command, **kwargs):
if self.config.has_option("unittest", "grep"):
grep = self.config.get("unittest", "grep")
else:
grep = "/bin/grep"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, grep)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
if p.returncode == 1:
return []
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(command, out, err))
def findcommand(self, command, **kwargs):
if self.config.has_option("unittest", "find"):
find = self.config.get("unittest", "find")
else:
find = "/usr/bin/find"
if isinstance(command, list):
args = command[:]
else:
args = [command]
args.insert(0, find)
env = {}
p = Popen(args, stdout=PIPE, stderr=PIPE, **kwargs)
(out, err) = p.communicate()
# Ignore out/err unless we get a non-zero return code, then log it.
if p.returncode == 0:
return out.splitlines()
self.fail("Error return code for %s, "
"STDOUT:\n@@@\n'%s'\n@@@\nSTDERR:\n@@@\n'%s'\n@@@\n" %
(command, out, err))
def writescratch(self, filename, contents):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'w') as f:
f.write(contents)
return scratchfile
def readscratch(self, filename):
scratchfile = os.path.join(self.scratchdir, filename)
with open(scratchfile, 'r') as f:
contents = f.read()
return contents
def dsdb_expect(self, command, fail=False, errstr=""):
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
if fail:
filename = DSDB_EXPECT_FAILURE_FILE
else:
filename = DSDB_EXPECT_SUCCESS_FILE
expected_name = os.path.join(dsdb_coverage_dir, filename)
with open(expected_name, "a") as fp:
if isinstance(command, list):
fp.write(" ".join([str(cmd) for cmd in command]))
else:
fp.write(str(command))
fp.write("\n")
if fail and errstr:
errfile = DSDB_EXPECT_FAILURE_ERROR
expected_name = os.path.join(dsdb_coverage_dir, errfile)
with open(expected_name, "a") as fp:
fp.write(errstr)
fp.write("\n")
def dsdb_expect_add(self,
hostname,
ip,
interface=None,
mac=None,
primary=None,
comments=None,
fail=False):
command = [
"add_host", "-host_name", hostname, "-ip_address",
str(ip), "-status", "aq"
]
if interface:
command.extend(
["-interface_name",
str(interface).replace('/', '_')])
if mac:
command.extend(["-ethernet_address", str(mac)])
if primary:
command.extend(["-primary_host_name", primary])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_delete(self, ip, fail=False):
self.dsdb_expect("delete_host -ip_address %s" % ip, fail=fail)
def dsdb_expect_update(self,
fqdn,
iface=None,
ip=None,
mac=None,
comments=None,
fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if iface:
command.extend(["-interface_name", iface])
if ip:
command.extend(["-ip_address", str(ip)])
if mac:
command.extend(["-ethernet_address", str(mac)])
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_rename(self,
fqdn,
new_fqdn=None,
iface=None,
new_iface=None,
fail=False):
command = ["update_aqd_host", "-host_name", fqdn]
if new_fqdn:
command.extend(["-primary_host_name", new_fqdn])
if iface:
command.extend(["-interface_name", iface])
if new_iface:
command.extend(["-new_interface_name", new_iface])
self.dsdb_expect(" ".join(command), fail=fail)
def dsdb_expect_add_campus(self,
campus,
comments=None,
fail=False,
errstr=""):
command = ["add_campus_aq", "-campus_name", campus]
if comments:
command.extend(["-comments", comments])
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus(self, campus, fail=False, errstr=""):
command = ["delete_campus_aq", "-campus", campus]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_add_campus_building(self,
campus,
building,
fail=False,
errstr=""):
command = [
"add_campus_building_aq", "-campus_name", campus, "-building_name",
building
]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_expect_del_campus_building(self,
campus,
building,
fail=False,
errstr=""):
command = [
"delete_campus_building_aq", "-campus_name", campus,
"-building_name", building
]
self.dsdb_expect(" ".join(command), fail=fail, errstr=errstr)
def dsdb_verify(self, empty=False):
dsdb_coverage_dir = os.path.join(
self.config.get("unittest", "scratchdir"), "dsdb_coverage")
fail_expected_name = os.path.join(dsdb_coverage_dir,
DSDB_EXPECT_FAILURE_FILE)
issued_name = os.path.join(dsdb_coverage_dir, DSDB_ISSUED_CMDS_FILE)
expected = {}
for filename in [DSDB_EXPECT_SUCCESS_FILE, DSDB_EXPECT_FAILURE_FILE]:
expected_name = os.path.join(dsdb_coverage_dir, filename)
try:
with open(expected_name, "r") as fp:
for line in fp:
expected[line.rstrip("\n")] = True
except IOError:
pass
# This is likely a logic error in the test
if not expected and not empty:
self.fail("dsdb_verify() called when no DSDB commands were "
"expected?!?")
issued = {}
try:
with open(issued_name, "r") as fp:
for line in fp:
issued[line.rstrip("\n")] = True
except IOError:
pass
errors = []
for cmd, dummy in expected.items():
if cmd not in issued:
errors.append("'%s'" % cmd)
# Unexpected DSDB commands are caught by the fake_dsdb script
if errors:
self.fail("The following expected DSDB commands were not called:"
"\n@@@\n%s\n@@@\n" % "\n".join(errors))
def verify_buildfiles(self,
domain,
object,
want_exist=True,
command='manage'):
qdir = self.config.get('broker', 'quattordir')
domaindir = os.path.join(qdir, 'build', 'xml', domain)
xmlfile = os.path.join(domaindir, object + self.profile_suffix)
depfile = os.path.join(domaindir, object + '.dep')
builddir = self.config.get('broker', 'builddir')
profile = os.path.join(builddir, 'domains', domain, 'profiles',
object + self.template_extension)
for f in [xmlfile, depfile, profile]:
if want_exist:
self.failUnless(
os.path.exists(f),
"Expecting %s to exist before running %s." % (f, command))
else:
self.failIf(
os.path.exists(f),
"Not expecting %s to exist after running %s." %
(f, command))
def demote_current_user(self, role="nobody"):
principal = self.config.get('unittest', 'principal')
command = ["permission", "--role", role, "--principal", principal]
self.noouttest(command)
def promote_current_user(self):
srcdir = self.config.get("broker", "srcdir")
add_admin = os.path.join(srcdir, "tests", "aqdb", "add_admin.py")
env = os.environ.copy()
env['AQDCONF'] = self.config.baseconfig
p = Popen([add_admin], stdout=PIPE, stderr=PIPE, env=env)
(out, err) = p.communicate()
self.assertEqual(
p.returncode, 0,
"Failed to restore admin privs '%s', '%s'." % (out, err))
class Plenary(object):
template_type = None
""" Specifies the PAN template type to generate """
handlers = {}
""" The handlers dictionary should have an entry for every subclass.
Typically this will be defined immediately after defining the
subclass.
"""
def __init__(self, dbobj=None, logger=LOGGER):
self.config = Config()
self.dbobj = dbobj
self.logger = logger
if self.template_type is None:
raise InternalError("Plenary class %s did not set the template "
"type" % self.__class__.__name__)
# Object templates live under the branch-specific build directory.
# Everything else lives under the common plenary directory.
if self.template_type == "object":
if not dbobj or not hasattr(dbobj, "branch"):
raise InternalError("Plenaries meant to be compiled need a DB "
"object that has a branch; got: %r" % dbobj)
self.dir = "%s/domains/%s/profiles" % (
self.config.get("broker", "builddir"), dbobj.branch.name)
else:
self.dir = self.config.get("broker", "plenarydir")
self.loadpath = None
self.plenary_template = None
self.plenary_core = None
self.new_content = None
# The following attributes are for stash/restore_stash
self.old_content = None
self.old_mtime = None
self.stashed = False
self.removed = False
self.changed = False
def __hash__(self):
"""Since equality is based on dbobj, just hash on it."""
return hash(self.dbobj)
def __eq__(self, other):
"""Plenary objects are equal if they describe the same object.
Technically this should probably also check that the class
matches. There are some odd cases when the plenary stores
extra information, currently ignored.
"""
if self.dbobj is None or other.dbobj is None:
return False
return self.dbobj == other.dbobj
def __str__(self):
"""For debug output."""
return "Plenary(%s)" % self.dbobj
@property
def plenary_directory(self):
""" Directory where the plenary template lives """
if self.loadpath and self.template_type != "object":
return "%s/%s/%s" % (self.dir, self.loadpath, self.plenary_core)
else:
return "%s/%s" % (self.dir, self.plenary_core)
@property
def plenary_file(self):
""" Full absolute path name of the plenary template """
return "%s/%s%s" % (self.plenary_directory, self.plenary_template,
TEMPLATE_EXTENSION)
@property
def plenary_template_name(self):
""" Name of the template as used by PAN, relative to the load path """
if self.plenary_core:
return "%s/%s" % (self.plenary_core, self.plenary_template)
else:
return self.plenary_template
def body(self, lines):
"""
The text of the template. By default, do nothing. A derived class can
override this to describe their own content.
They should do this by appending strings (each string
referring to a separate line of text in the template) into the
array. The array will already contain the appropriate header line for the
template.
"""
pass
def will_change(self):
self.stash()
if not self.new_content:
self.new_content = self._generate_content()
return self.old_content != self.new_content
def get_write_key(self):
if self.will_change():
return self.get_key()
return None
def get_remove_key(self):
"""Return the relevant key.
In many cases it will be more efficient just to create a full
compile lock and bypass this method.
"""
return self.get_key()
def get_key(self):
"""Base implementation assumes a full compile lock."""
return CompileKey(logger=self.logger)
def _generate_content(self):
"""Not meant to be overridden or called directly."""
lines = []
type = self.template_type
if type is not None and type is not "":
type = type + " "
lines.append("%stemplate %s;" % (type, self.plenary_template_name))
lines.append("")
if self.template_type == "object":
if self.loadpath:
pan_variable(lines, "LOADPATH", [self.loadpath])
lines.append("")
pan_assign(lines, "/metadata/template/branch/name",
self.dbobj.branch.name)
pan_assign(lines, "/metadata/template/branch/type",
self.dbobj.branch.branch_type)
if self.dbobj.branch.branch_type == 'sandbox':
pan_assign(lines, "/metadata/template/branch/author",
self.dbobj.sandbox_author.name)
lines.append("")
self.body(lines)
return "\n".join(lines) + "\n"
def write(self, locked=False, content=None):
"""Write out the template.
If the content is unchanged, then the file will not be modified
(preserving the mtime).
Returns the number of files that were written.
If locked is True then it is assumed that error handling happens
higher in the call stack.
"""
if self.template_type == "object" and \
hasattr(self.dbobj, "personality") and \
self.dbobj.personality and \
not self.dbobj.personality.archetype.is_compileable:
return 0
if content is None:
if not self.new_content:
self.new_content = self._generate_content()
content = self.new_content
self.stash()
if self.old_content == content and \
not self.removed and not self.changed:
# optimise out the write (leaving the mtime good for ant)
# if nothing is actually changed
return 0
key = None
try:
if not locked:
key = self.get_write_key()
lock_queue.acquire(key)
if not os.path.exists(self.plenary_directory):
os.makedirs(self.plenary_directory)
write_file(self.plenary_file, content, logger=self.logger)
self.removed = False
if self.old_content != content:
self.changed = True
except Exception, e:
if not locked:
self.restore_stash()
raise e
finally:
class BrokerCommand(object):
""" The basis for each command module under commands.
Several class-level lists and flags are defined here that can be
overridden on a per-command basis. Some can only be overridden
in __init__, though, so check the docstrings.
"""
required_parameters = []
""" This will generally be overridden in the command class.
It could theoretically be parsed out of input.xml, but that is
tricky in some cases and possibly error-prone.
"""
optional_parameters = []
""" Optional parameters are filled in automatically from input.xml.
This may contain entries that are also in the required_parameters list.
If so, the required entry "wins".
"""
requires_azcheck = True
""" Opt out of authorization checks by setting this flag to False."""
requires_transaction = True
""" This sets up a session and cleans up when finished.
Currently, setting azcheck to True will force this value to be True,
because the azcheck requires a session that will need to be cleaned
up.
"""
requires_format = False
""" Run command results through the formatter.
It is automatically set to True for all cat, search, and show
commands, but could be reversed back to False by overriding __init__
for the command.
"""
requires_readonly = False
""" Require read only isolation level for the render session.
It is automatically set to True for all search and show commands,
but could be reversed back to False by overriding __init__ for the
command.
"""
# Override to indicate whether the command will generally take a
# lock during execution.
#
# Any command with this flag set to True will use the separate
# NLSession thread pool and should not have to wait on commands
# that are potentially all blocking on the same lock.
#
# If set to None (the default), the is_lock_free property will
# examine the command's module to try to determine if a lock
# may be required and then cache the value.
_is_lock_free = None
# Run the render method on a separate thread. This will be forced
# to True if requires_azcheck or requires_transaction.
defer_to_thread = True
def __init__(self):
""" Provides some convenient variables for commands.
Also sets requires_* parameters for some sets of commands.
All of the command objects are singletons (or Borg).
"""
self.dbf = DbFactory()
self.config = Config()
self.az = AuthorizationBroker()
self.formatter = ResponseFormatter()
self.catalog = StatusCatalog()
# Force the instance to have local copies of the class defaults...
# This allows resources.py to modify instances without worrying
# about inheritance issues (classes sharing required or optional
# parameters).
self.required_parameters = self.required_parameters[:]
self.optional_parameters = self.optional_parameters[:]
# Parameter checks are filled in automatically based on input.xml. This
# lets us do some rudimentary checks before the actual command is
# invoked.
self.parameter_checks = {}
# The parameter types are filled in automatically based on input.xml.
self.parameter_types = {}
# This is the pivot of the above, filled in at the same time. It is a
# dictionary of type names to lists of parameters.
self.parameters_by_type = {}
self.action = self.__module__
package_prefix = "aquilon.worker.commands."
if self.action.startswith(package_prefix):
self.action = self.action[len(package_prefix):]
# self.command is set correctly in resources.py after parsing input.xml
self.command = self.action
# The readonly and format flags are done here for convenience
# and simplicity. They could be overridden by the __init__
# method of any show/search/cat commands that do not want these
# defaults. Some 'one-off' commands (like ping and status)
# just set the variables themselves.
if self.action.startswith("show") or self.action.startswith("search"):
self.requires_readonly = True
self.requires_format = True
if self.action.startswith("cat"):
self.requires_format = True
self.requires_readonly = True
self._is_lock_free = True
if not self.requires_readonly \
and self.config.get('broker', 'mode') == 'readonly':
self.badmode = 'readonly'
else:
self.badmode = False
self._update_render(self.render)
if not self.defer_to_thread:
if self.requires_azcheck or self.requires_transaction:
self.defer_to_thread = True
log.msg("Forcing defer_to_thread to True because of "
"required authorization or transaction for %s" %
self.command)
# Not sure how to handle formatting with deferred...
self.requires_format = False
#free = "True " if self.is_lock_free else "False"
#log.msg("is_lock_free = %s [%s]" % (free, self.command))
def audit_result(self, session, key, value, **arguments):
# We need a place to store the result somewhere until we can finish the
# audit record. Use the request object for now.
request = arguments["request"]
if not hasattr(request, "_audit_result"):
request._audit_result = []
request._audit_result.append((key, value))
def render(self, **arguments): # pragma: no cover
""" Implement this method to create a functional broker command.
The base __init__ method wraps all implementations using
_update_render() to enforce the class requires_* flags.
"""
if self.__class__.__module__ == 'aquilon.worker.broker':
# Default class... no useful command info to repeat back...
raise UnimplementedError("Command has not been implemented.")
raise UnimplementedError("%s has not been implemented" %
self.__class__.__module__)
def _update_render(self, command):
""" Wrap the render method using the requires_* attributes.
An alternate implementation would be to just have a
wrap_rendor() method or similar that got called instead
of rendor().
"""
def updated_render(self, *args, **kwargs):
principal = kwargs["user"]
request = kwargs["request"]
logger = kwargs["logger"]
raising_exception = None
rollback_failed = False
try:
if self.requires_transaction or self.requires_azcheck:
# Set up a session...
if not "session" in kwargs:
if self.is_lock_free:
kwargs["session"] = self.dbf.NLSession()
else:
kwargs["session"] = self.dbf.Session()
session = kwargs["session"]
if session.bind.dialect.name == "oracle":
# Make the name of the command and the request ID
# available in v$session. Trying to set a value longer
# than the allowed length will generate ORA-24960, so
# do an explicit truncation.
conn = session.connection()
dbapi_con = conn.connection.connection
dbapi_con.action = str(self.action)[:32]
# TODO: we should include the command number as well,
# since that is easier to find in the logs
dbapi_con.clientinfo = str(kwargs["requestid"])[:64]
# This does a COMMIT, which in turn invalidates the session.
# We should therefore avoid looking up anything in the DB
# before this point which might be used later.
self._record_xtn(session, logger.get_status())
dbuser = get_or_create_user_principal(session,
principal,
commitoncreate=True)
kwargs["dbuser"] = dbuser
if self.requires_azcheck:
self.az.check(principal=principal,
dbuser=dbuser,
action=self.action,
resource=request.path)
if self.requires_readonly:
self._set_readonly(session)
# begin() is only required if session transactional=False
#session.begin()
if self.badmode: # pragma: no cover
raise UnimplementedError("Command %s not available on "
"a %s broker." %
(self.command, self.badmode))
for key in kwargs.keys():
if key in self.parameter_checks:
kwargs[key] = self.parameter_checks[key]("--" + key,
kwargs[key])
# Command is an instance method already having self...
retval = command(*args, **kwargs)
if self.requires_format:
style = kwargs.get("style", None)
retval = self.formatter.format(style, retval, request)
if "session" in kwargs:
session.commit()
return retval
except Exception, e:
raising_exception = e
# Need to close after the rollback, or the next time session
# is accessed it tries to commit the transaction... (?)
if "session" in kwargs:
try:
session.rollback()
except: # pragma: no cover
rollback_failed = True
raise
session.close()
raise
finally:
def makeService(self, options):
# Start up coverage ASAP.
coverage_dir = options["coveragedir"]
if coverage_dir:
os.makedirs(coverage_dir, 0755)
if options["coveragerc"]:
coveragerc = options["coveragerc"]
else:
coveragerc = None
self.coverage = coverage.coverage(config_file=coveragerc)
self.coverage.erase()
self.coverage.start()
# Get the config object.
config = Config(configfile=options["config"])
# Helper for finishing off the coverage report.
def stop_coverage():
log.msg("Finishing coverage")
self.coverage.stop()
aquilon_srcdir = os.path.join(config.get("broker", "srcdir"),
"lib", "python2.6", "aquilon")
sourcefiles = []
for dirpath, dirnames, filenames in os.walk(aquilon_srcdir):
# FIXME: try to do this from the coverage config file
if dirpath.endswith("aquilon"):
dirnames.remove("client")
elif dirpath.endswith("aqdb"):
dirnames.remove("utils")
for filename in filenames:
if not filename.endswith('.py'):
continue
sourcefiles.append(os.path.join(dirpath, filename))
self.coverage.html_report(sourcefiles, directory=coverage_dir)
self.coverage.xml_report(sourcefiles,
outfile=os.path.join(
coverage_dir, "aqd.xml"))
with open(os.path.join(coverage_dir, "aqd.coverage"),
"w") as outfile:
self.coverage.report(sourcefiles, file=outfile)
# Make sure the coverage report gets generated.
if coverage_dir:
reactor.addSystemEventTrigger('after', 'shutdown', stop_coverage)
# Set up the environment...
m = Modulecmd()
log_module_load(m, config.get("broker", "CheckNet_module"))
if config.has_option("database", "module"):
log_module_load(m, config.get("database", "module"))
sys.path.append(config.get("protocols", "directory"))
# Set this up before the aqdb libs get imported...
integrate_logging(config)
progname = os.path.split(sys.argv[0])[1]
if progname == 'aqd':
if config.get('broker', 'mode') != 'readwrite':
log.msg("Broker started with aqd symlink, "
"setting config mode to readwrite")
config.set('broker', 'mode', 'readwrite')
if progname == 'aqd_readonly':
if config.get('broker', 'mode') != 'readonly':
log.msg("Broker started with aqd_readonly symlink, "
"setting config mode to readonly")
config.set('broker', 'mode', 'readonly')
log.msg("Loading broker in mode %s" % config.get('broker', 'mode'))
# Dynamic import means that we can parse config options before
# importing aqdb. This is a hack until aqdb can be imported without
# firing up database connections.
resources = __import__("aquilon.worker.resources", globals(), locals(),
["RestServer"], -1)
RestServer = getattr(resources, "RestServer")
restServer = RestServer(config)
openSite = AnonSite(restServer)
# twisted is nicely changing the umask for us when the process is
# set to daemonize. This sets it back.
restServer.set_umask()
reactor.addSystemEventTrigger('after', 'startup', restServer.set_umask)
reactor.addSystemEventTrigger('after', 'startup',
restServer.set_thread_pool_size)
sockdir = config.get("broker", "sockdir")
if not os.path.exists(sockdir):
os.makedirs(sockdir, 0700)
os.chmod(sockdir, 0700)
if options["usesock"]:
return strports.service("unix:%s/aqdsock" % sockdir, openSite)
openport = config.get("broker", "openport")
if config.has_option("broker", "bind_address"):
bind_address = config.get("broker", "bind_address").strip()
openaddr = "tcp:%s:interface=%s" % (openport, bind_address)
else: # pragma: no cover
bind_address = None
openaddr = "tcp:%s" % openport
# Return before firing up knc.
if options["noauth"]:
return strports.service(openaddr, openSite)
sockname = os.path.join(sockdir, "kncsock")
# This flag controls whether or not this process will start up
# and monitor knc. Except for noauth mode knc has to be running,
# but this process doesn't have to be the thing that starts it up.
if config.getboolean("broker", "run_knc") or \
config.getboolean("broker", "run_git_daemon"):
mon = GracefulProcessMonitor()
# FIXME: Should probably run krb5_keytab here as well.
# and/or verify that the keytab file exists.
if config.getboolean("broker", "run_knc"):
keytab = config.get("broker", "keytab")
knc_args = [
"/usr/bin/env",
"KRB5_KTNAME=FILE:%s" % keytab,
config.get("kerberos", "knc"), "-lS", sockname
]
if bind_address:
knc_args.append("-a")
knc_args.append(bind_address)
knc_args.append(config.get("broker", "kncport"))
mon.addProcess("knc", knc_args)
if config.getboolean("broker", "run_git_daemon"):
# The git daemon *must* be invoked using the form 'git-daemon'
# instead of invoking git with a 'daemon' argument. The latter
# will fork and exec git-daemon, resulting in a new pid that
# the process monitor won't know about!
gitpath = config.get("broker", "git_path")
gitdaemon = config.get("broker", "git_daemon")
ospath = os.environ.get("PATH", "")
args = [
"/usr/bin/env",
"PATH=%s:%s" % (gitpath, ospath), gitdaemon,
"--export-all",
"--base-path=%s" %
config.get("broker", "git_daemon_basedir")
]
if config.has_option("broker", "git_port"):
args.append("--port=%s" % config.get("broker", "git_port"))
if bind_address:
args.append("--listen=%s" % bind_address)
args.append(config.get("broker", "kingdir"))
mon.addProcess("git-daemon", args)
mon.startService()
reactor.addSystemEventTrigger('before', 'shutdown',
mon.stopService)
# This socket is created by twisted and only accessed by knc as
# connections come in.
if os.path.exists(sockname):
try:
log.msg("Attempting to remove old socket '%s'" % sockname)
os.remove(sockname)
log.msg("Succeeded removing old socket.")
except OSError, e:
log.msg("Could not remove old socket '%s': %s" % (sockname, e))
