def render(self, session, logger, dbuser, sandbox, start, get, comments, **arguments):
if not dbuser:
raise AuthorizationException("Cannot create a sandbox without an " "authenticated connection.")
sandbox = self.force_my_sandbox(session, logger, dbuser, sandbox)
# See `git check-ref-format --help` for naming restrictions.
# We want to layer a few extra restrictions on top of that...
validate_template_name("--sandbox", sandbox)
Branch.get_unique(session, sandbox, preclude=True)
# Check that the user has cleared up a directory of the same
# name; if this is not the case the branch may be created (in git)
# and added to the database - however CommandGet will fail roleing
# back the database leaving the branch created in git
templatesdir = self.config.get("broker", "templatesdir")
sandboxdir = os.path.join(templatesdir, dbuser.name, sandbox)
if os.path.exists(sandboxdir):
raise ArgumentError("Sandbox directory %s already exists; " "cannot create branch." % sandboxdir)
if not start:
start = self.config.get("broker", "default_domain_start")
dbstart = Branch.get_unique(session, start, compel=True)
kingdir = self.config.get("broker", "kingdir")
base_commit = run_git(["show-ref", "--hash", "refs/heads/" + dbstart.name], logger=logger, path=kingdir)
compiler = self.config.get("panc", "pan_compiler")
dbsandbox = Sandbox(name=sandbox, owner=dbuser, compiler=compiler, base_commit=base_commit, comments=comments)
session.add(dbsandbox)
session.flush()
# Currently this will fail if the branch already exists...
# That seems like the right behavior. It's an internal
# consistency issue that would need to be addressed explicitly.
run_git(["branch", sandbox, dbstart.name], logger=logger, path=kingdir)
# If we arrive there the above "git branch" command has succeeded;
# therefore we should comit the changes to the database. If this is
# not done, and CommandGet fails (see dir check above), then the
# git branch will be created but the database changes roled back.
session.commit()
if get == False:
# The client knows to interpret an empty response as no action.
return []
return CommandGet.render(self, session=session, logger=logger, dbuser=dbuser, sandbox=sandbox)
def test_create_hosts():
br = Branch.get_unique(sess, 'ny-prod', compel=True)
dns_dmn = DnsDomain.get_unique(sess, 'one-nyp.ms.com', compel=True)
stat = Status.get_unique(sess, 'build', compel=True)
os = sess.query(OperatingSystem).filter(Archetype.name == 'vmhost').first()
assert os, 'No OS in %s' % func_name()
pers = sess.query(Personality).select_from(join(
Personality, Archetype)).filter(
and_(Archetype.name == 'vmhost',
Personality.name == 'generic')).one()
sess.autoflush = False
for i in xrange(NUM_HOSTS):
machine = m_factory.next()
vm_host = Host(machine=machine,
name='%s%s' % (HOST_NAME, i),
dns_domain=dns_dmn,
branch=br,
personality=pers,
status=stat,
operating_system=os)
add(sess, vm_host)
sess.autoflush = True
commit(sess)
hosts = sess.query(Host).filter(Host.name.like(HOST_NAME + '%')).all()
assert len(hosts) is NUM_HOSTS
print 'created %s hosts' % len(hosts)
def test_create_hosts():
br = Branch.get_unique(sess, 'ny-prod', compel=True)
dns_dmn = DnsDomain.get_unique(sess, 'one-nyp.ms.com', compel=True)
stat = Status.get_unique(sess, 'build', compel=True)
os = sess.query(OperatingSystem).filter(Archetype.name == 'vmhost').first()
assert os, 'No OS in %s' % func_name()
pers = sess.query(Personality).select_from(
join(Personality, Archetype)).filter(
and_(Archetype.name=='vmhost', Personality.name=='generic')).one()
sess.autoflush=False
for i in xrange(NUM_HOSTS):
machine = m_factory.next()
vm_host = Host(machine=machine, name='%s%s' % (HOST_NAME, i),
dns_domain=dns_dmn, branch=br, personality=pers,
status=stat, operating_system=os)
add(sess, vm_host)
sess.autoflush=True
commit(sess)
hosts = sess.query(Host).filter(
Host.name.like(HOST_NAME+'%')).all()
assert len(hosts) is NUM_HOSTS
print 'created %s hosts'% len(hosts)
def render(self, session, logger, branch, comments, **arguments):
dbbranch = Branch.get_unique(session, branch, compel=True)
if dbbranch.is_sync_valid:
logger.warn("{0} already marked as valid.".format(dbbranch))
# For now, comments just get stored in the audit log.
dbbranch.is_sync_valid = True
session.flush()
# FIXME: Run tests?
return
def render(self, session, logger, branch, comments, **arguments):
dbbranch = Branch.get_unique(session, branch, compel=True)
if dbbranch.is_sync_valid:
logger.warn("{0} already marked as valid.".format(dbbranch))
# For now, comments just get stored in the audit log.
dbbranch.is_sync_valid = True
session.add(dbbranch)
# FIXME: Run tests?
return
def render(self, session, logger, dbuser, sandbox, start, get, comments,
**arguments):
if not dbuser:
raise AuthorizationException("Cannot create a sandbox without an "
"authenticated connection.")
sandbox = self.force_my_sandbox(session, logger, dbuser, sandbox)
# See `git check-ref-format --help` for naming restrictions.
# We want to layer a few extra restrictions on top of that...
valid = re.compile('^[a-zA-Z0-9_.-]+$')
if (not valid.match(sandbox)):
raise ArgumentError("sandbox name '%s' is not valid" % sandbox)
Branch.get_unique(session, sandbox, preclude=True)
if not start:
start = self.config.get("broker", "default_domain_start")
dbstart = Branch.get_unique(session, start, compel=True)
kingdir = self.config.get("broker", "kingdir")
base_commit = run_git(["show-ref", "--hash", "refs/heads/" +
dbstart.name], logger=logger, path=kingdir)
compiler = self.config.get("panc", "pan_compiler")
dbsandbox = Sandbox(name=sandbox, owner=dbuser, compiler=compiler,
base_commit=base_commit, comments=comments)
session.add(dbsandbox)
session.flush()
# Currently this will fail if the branch already exists...
# That seems like the right behavior. It's an internal
# consistency issue that would need to be addressed explicitly.
run_git(["branch", sandbox, dbstart.name], logger=logger, path=kingdir)
if get == False:
# The client knows to interpret an empty response as no action.
return []
return CommandGet.render(self, session=session, logger=logger,
dbuser=dbuser, sandbox=sandbox)
def test_no_host_threshold():
""" ensure down_hosts_threshold must exist """
br = Branch.get_unique(sess, 'ny-prod', compel=True)
np = Building.get_unique(sess, name='np', compel=True)
per = sess.query(Personality).select_from(
join(Archetype, Personality)).filter(
and_(Archetype.name=='windows', Personality.name=='generic')).one()
ec = EsxCluster(name=CLUSTER_NAME,
location_constraint=np, personality=per, branch=br)
add(sess, ec)
commit(sess)
def test_no_host_threshold():
""" ensure down_hosts_threshold must exist """
br = Branch.get_unique(sess, 'ny-prod', compel=True)
np = Building.get_unique(sess, name='np', compel=True)
per = sess.query(Personality).select_from(join(
Archetype, Personality)).filter(
and_(Archetype.name == 'windows',
Personality.name == 'generic')).one()
ec = EsxCluster(name=CLUSTER_NAME,
location_constraint=np,
personality=per,
branch=br)
add(sess, ec)
commit(sess)
def render(self, session, track, not_tracking, change_manager, fullinfo, **arguments):
q = search_branch_query(self.config, session, Domain, **arguments)
if track:
dbtracked = Branch.get_unique(session, track, compel=True)
q = q.filter_by(tracked_branch=dbtracked)
if not_tracking:
q = q.filter_by(tracked_branch=None)
if change_manager is not None:
q = q.filter_by(requires_change_manager=change_manager)
q = q.order_by(Domain.name)
result = q.all()
if fullinfo:
return result
else:
return StringList(result)
def test_create_esx_cluster():
""" tests the creation of an EsxCluster """
np = Building.get_unique(sess, name='np', compel=True)
br = Branch.get_unique(sess, 'ny-prod', compel=True)
per = sess.query(Personality).select_from(
join(Archetype, Personality)).filter(
and_(Archetype.name=='windows', Personality.name=='generic')).one()
ec = EsxCluster(name=CLUSTER_NAME, location_constraint=np,
personality=per, down_hosts_threshold=2, branch=br)
create(sess, ec)
assert ec
print ec
assert ec.max_hosts is 8
print 'esx cluster max members = %s' % ec.max_hosts
def render(self, session, track, not_tracking, change_manager, fullinfo,
**arguments):
q = search_branch_query(self.config, session, Domain, **arguments)
if track:
dbtracked = Branch.get_unique(session, track, compel=True)
q = q.filter_by(tracked_branch=dbtracked)
if not_tracking:
q = q.filter_by(tracked_branch=None)
if change_manager is not None:
q = q.filter_by(requires_change_manager=change_manager)
q = q.order_by(Domain.name)
result = q.all()
if fullinfo:
return result
else:
return StringList(result)
def get_branch_and_author(session, logger,
domain=None, sandbox=None, branch=None,
compel=False):
dbbranch = None
dbauthor = None
if domain:
dbbranch = Domain.get_unique(session, domain, compel=True)
elif branch:
dbbranch = Branch.get_unique(session, branch, compel=True)
elif sandbox:
(author, slash, name) = sandbox.partition('/')
if not slash:
raise ArgumentError("Expected sandbox as 'author/branch', author "
"name and branch name separated by a slash.")
dbbranch = Sandbox.get_unique(session, name, compel=True)
dbauthor = get_user_principal(session, author)
elif compel:
raise ArgumentError("Please specify either sandbox or domain.")
return (dbbranch, dbauthor)
def test_create_esx_cluster():
""" tests the creation of an EsxCluster """
np = Building.get_unique(sess, name='np', compel=True)
br = Branch.get_unique(sess, 'ny-prod', compel=True)
per = sess.query(Personality).select_from(join(
Archetype, Personality)).filter(
and_(Archetype.name == 'windows',
Personality.name == 'generic')).one()
ec = EsxCluster(name=CLUSTER_NAME,
location_constraint=np,
personality=per,
down_hosts_threshold=2,
branch=br)
create(sess, ec)
assert ec
print ec
assert ec.max_hosts is 8
print 'esx cluster max members = %s' % ec.max_hosts
def get_branch_and_author(session,
logger,
domain=None,
sandbox=None,
branch=None,
compel=False):
dbbranch = None
dbauthor = None
if domain:
dbbranch = Domain.get_unique(session, domain, compel=True)
elif branch:
dbbranch = Branch.get_unique(session, branch, compel=True)
elif sandbox:
(author, slash, name) = sandbox.partition('/')
if not slash:
raise ArgumentError("Expected sandbox as 'author/branch', author "
"name and branch name separated by a slash.")
dbbranch = Sandbox.get_unique(session, name, compel=True)
dbauthor = get_user_principal(session, author)
elif compel:
raise ArgumentError("Please specify either sandbox or domain.")
return (dbbranch, dbauthor)
def test_create_clusters():
np = sess.query(Building).filter_by(name='np').one()
br = Branch.get_unique(sess, 'ny-prod', compel=True)
per = sess.query(Personality).select_from(
join(Archetype, Personality)).filter(
and_(Archetype.name == 'windows',
Personality.name == 'generic')).one()
for i in xrange(NUM_CLUSTERS):
ec = EsxCluster(name='%s%s' % (CLUSTER_NAME, i),
location_constraint=np, branch=br,
personality=per, down_hosts_threshold=2)
add(sess, ec)
commit(sess)
ecs = sess.query(EsxCluster).all()
assert len(ecs) is NUM_CLUSTERS
print ecs[0]
assert ecs[0].max_hosts is 8
print 'esx cluster max hosts = %s' % (ecs[0].max_hosts)
def render(self, session, logger, source, target, sync, dryrun,
comments, justification, user, requestid, **arguments):
# Most of the logic here is duplicated in publish
dbsource = Branch.get_unique(session, source, compel=True)
# The target has to be a non-tracking domain
dbtarget = Domain.get_unique(session, target, compel=True)
if sync and isinstance(dbtarget.tracked_branch, Domain) \
and dbtarget.tracked_branch.autosync and dbtarget.autosync:
# The user probably meant to deploy to the tracked branch,
# but only do so if all the relevant autosync flags are
# positive.
logger.warning("Deploying to tracked branch %s and then will "
"auto-sync %s" % (
dbtarget.tracked_branch.name, dbtarget.name))
dbtarget = dbtarget.tracked_branch
elif dbtarget.tracked_branch:
raise ArgumentError("Cannot deploy to tracking domain %s. "
"Did you mean domain %s?" %
(dbtarget.name, dbtarget.tracked_branch.name))
if sync and not dbtarget.is_sync_valid and dbtarget.trackers:
# FIXME: Maybe raise an ArgumentError and request that the
# command run with --nosync? Maybe provide a --validate flag?
# For now, just auto-flip (below).
pass
if not dbtarget.is_sync_valid:
dbtarget.is_sync_valid = True
if dbtarget.requires_change_manager:
if not justification:
raise AuthorizationException(
"{0} is under change management control. Please specify "
"--justification.".format(dbtarget))
validate_justification(user, justification)
if isinstance(dbsource, Sandbox):
domainsdir = self.config.get('broker', 'domainsdir')
targetdir = os.path.join(domainsdir, dbtarget.name)
filterre = re.compile('^' + dbsource.base_commit + '$')
found = run_git(['rev-list', 'HEAD'], path=targetdir,
logger=logger, filterre=filterre)
if not found:
raise ArgumentError("You're trying to deploy a sandbox to a "
"domain that does not contain the commit "
"where the sandbox was branched from.")
kingdir = self.config.get("broker", "kingdir")
rundir = self.config.get("broker", "rundir")
tempdir = mkdtemp(prefix="deploy_", suffix="_%s" % dbsource.name,
dir=rundir)
try:
run_git(["clone", "--shared", "--branch", dbtarget.name,
kingdir, dbtarget.name],
path=tempdir, logger=logger)
temprepo = os.path.join(tempdir, dbtarget.name)
# We could try to use fmt-merge-msg but its usage is so obscure that
# faking it is easier
merge_msg = []
merge_msg.append("Merge remote branch 'origin/%s' into %s" %
(dbsource.name, dbtarget.name))
merge_msg.append("")
merge_msg.append("User: %s" % user)
merge_msg.append("Request ID: %s" % requestid)
if justification:
merge_msg.append("Justification: %s" % justification)
if comments:
merge_msg.append("Comments: %s" % comments)
try:
run_git(["merge", "--no-ff", "origin/%s" % dbsource.name,
"-m", "\n".join(merge_msg)],
path=temprepo, logger=logger, loglevel=CLIENT_INFO)
except ProcessException, e:
# No need to re-print e, output should have gone to client
# immediately via the logger.
raise ArgumentError("Failed to merge changes from %s into %s" %
(dbsource.name, dbtarget.name))
# FIXME: Run tests before pushing back to template-king.
# Use a different try/except and a specific error message.
if dryrun:
session.rollback()
return
run_git(["push", "origin", dbtarget.name],
path=temprepo, logger=logger)
def render(self, session, logger, dbuser, domain, track, start,
change_manager, comments, allow_manage, **arguments):
if not dbuser:
raise AuthorizationException("Cannot create a domain without "
"an authenticated connection.")
Branch.get_unique(session, domain, preclude=True)
valid = re.compile('^[a-zA-Z0-9_.-]+$')
if (not valid.match(domain)):
raise ArgumentError("Domain name '%s' is not valid." % domain)
# FIXME: Verify that track is a valid branch name?
# Or just let the branch command fail?
compiler = self.config.get("panc", "pan_compiler")
dbtracked = None
if track:
dbtracked = Branch.get_unique(session, track, compel=True)
if getattr(dbtracked, "tracked_branch", None):
raise ArgumentError("Cannot nest tracking. Try tracking "
"{0:l} directly.".format(dbtracked.tracked_branch))
start_point = dbtracked
if change_manager:
raise ArgumentError("Cannot enforce a change manager for "
"tracking domains.")
else:
if not start:
start = self.config.get("broker", "default_domain_start")
start_point = Branch.get_unique(session, start, compel=True)
dbdomain = Domain(name=domain, owner=dbuser, compiler=compiler,
tracked_branch=dbtracked,
requires_change_manager=bool(change_manager),
comments=comments)
session.add(dbdomain)
if allow_manage is not None:
dbdomain.allow_manage = allow_manage
session.flush()
domainsdir = self.config.get("broker", "domainsdir")
clonedir = os.path.join(domainsdir, dbdomain.name)
if os.path.exists(clonedir):
raise InternalError("Domain directory already exists")
kingdir = self.config.get("broker", "kingdir")
cmd = ["branch"]
if track:
cmd.append("--track")
else:
cmd.append("--no-track")
cmd.append(dbdomain.name)
cmd.append(start_point.name)
run_git(cmd, path=kingdir, logger=logger)
# If the branch command above fails the DB will roll back as normal.
# If the command below fails we need to clean up from itself and above.
try:
run_git(["clone", "--branch", dbdomain.name,
kingdir, dbdomain.name],
path=domainsdir, logger=logger)
except ProcessException, e:
try:
remove_dir(clonedir, logger=logger)
run_git(["branch", "-D", dbdomain.name],
path=kingdir, logger=logger)
except ProcessException, e2:
logger.info("Exception while cleaning up: %s", e2)
def render(self, session, logger, dbuser, domain, track, start,
change_manager, comments, allow_manage, **arguments):
if not dbuser:
raise AuthorizationException("Cannot create a domain without "
"an authenticated connection.")
Branch.get_unique(session, domain, preclude=True)
valid = re.compile('^[a-zA-Z0-9_.-]+$')
if (not valid.match(domain)):
raise ArgumentError("Domain name '%s' is not valid." % domain)
# FIXME: Verify that track is a valid branch name?
# Or just let the branch command fail?
compiler = self.config.get("panc", "pan_compiler")
dbtracked = None
if track:
dbtracked = Branch.get_unique(session, track, compel=True)
if getattr(dbtracked, "tracked_branch", None):
raise ArgumentError("Cannot nest tracking. Try tracking "
"{0:l} directly.".format(
dbtracked.tracked_branch))
start_point = dbtracked
if change_manager:
raise ArgumentError("Cannot enforce a change manager for "
"tracking domains.")
else:
if not start:
start = self.config.get("broker", "default_domain_start")
start_point = Branch.get_unique(session, start, compel=True)
dbdomain = Domain(name=domain,
owner=dbuser,
compiler=compiler,
tracked_branch=dbtracked,
requires_change_manager=bool(change_manager),
comments=comments)
session.add(dbdomain)
if allow_manage is not None:
dbdomain.allow_manage = allow_manage
session.flush()
domainsdir = self.config.get("broker", "domainsdir")
clonedir = os.path.join(domainsdir, dbdomain.name)
if os.path.exists(clonedir):
raise InternalError("Domain directory already exists")
kingdir = self.config.get("broker", "kingdir")
cmd = ["branch"]
if track:
cmd.append("--track")
else:
cmd.append("--no-track")
cmd.append(dbdomain.name)
cmd.append(start_point.name)
run_git(cmd, path=kingdir, logger=logger)
# If the branch command above fails the DB will roll back as normal.
# If the command below fails we need to clean up from itself and above.
try:
run_git(
["clone", "--branch", dbdomain.name, kingdir, dbdomain.name],
path=domainsdir,
logger=logger)
except ProcessException, e:
try:
remove_dir(clonedir, logger=logger)
run_git(["branch", "-D", dbdomain.name],
path=kingdir,
logger=logger)
except ProcessException, e2:
logger.info("Exception while cleaning up: %s", e2)
def render(self, session, logger, source, target, sync, dryrun, comments,
justification, user, requestid, **arguments):
# Most of the logic here is duplicated in publish
dbsource = Branch.get_unique(session, source, compel=True)
# The target has to be a non-tracking domain
dbtarget = Domain.get_unique(session, target, compel=True)
if sync and isinstance(dbtarget.tracked_branch, Domain) \
and dbtarget.tracked_branch.autosync and dbtarget.autosync:
# The user probably meant to deploy to the tracked branch,
# but only do so if all the relevant autosync flags are
# positive.
logger.warning("Deploying to tracked branch %s and then will "
"auto-sync %s" %
(dbtarget.tracked_branch.name, dbtarget.name))
dbtarget = dbtarget.tracked_branch
elif dbtarget.tracked_branch:
raise ArgumentError("Cannot deploy to tracking domain %s. "
"Did you mean domain %s?" %
(dbtarget.name, dbtarget.tracked_branch.name))
if sync and not dbtarget.is_sync_valid and dbtarget.trackers:
# FIXME: Maybe raise an ArgumentError and request that the
# command run with --nosync? Maybe provide a --validate flag?
# For now, just auto-flip (below).
pass
if not dbtarget.is_sync_valid:
dbtarget.is_sync_valid = True
if dbtarget.requires_change_manager:
if not justification:
raise AuthorizationException(
"{0} is under change management control. Please specify "
"--justification.".format(dbtarget))
validate_justification(user, justification)
if isinstance(dbsource, Sandbox):
domainsdir = self.config.get('broker', 'domainsdir')
targetdir = os.path.join(domainsdir, dbtarget.name)
filterre = re.compile('^' + dbsource.base_commit + '$')
found = run_git(['rev-list', 'HEAD'],
path=targetdir,
logger=logger,
filterre=filterre)
if not found:
raise ArgumentError("You're trying to deploy a sandbox to a "
"domain that does not contain the commit "
"where the sandbox was branched from.")
kingdir = self.config.get("broker", "kingdir")
rundir = self.config.get("broker", "rundir")
tempdir = mkdtemp(prefix="deploy_",
suffix="_%s" % dbsource.name,
dir=rundir)
try:
run_git([
"clone", "--shared", "--branch", dbtarget.name, kingdir,
dbtarget.name
],
path=tempdir,
logger=logger)
temprepo = os.path.join(tempdir, dbtarget.name)
# We could try to use fmt-merge-msg but its usage is so obscure that
# faking it is easier
merge_msg = []
merge_msg.append("Merge remote branch 'origin/%s' into %s" %
(dbsource.name, dbtarget.name))
merge_msg.append("")
merge_msg.append("User: %s" % user)
merge_msg.append("Request ID: %s" % requestid)
if justification:
merge_msg.append("Justification: %s" % justification)
if comments:
merge_msg.append("Comments: %s" % comments)
try:
run_git([
"merge", "--no-ff",
"origin/%s" % dbsource.name, "-m", "\n".join(merge_msg)
],
path=temprepo,
logger=logger,
loglevel=CLIENT_INFO)
except ProcessException, e:
# No need to re-print e, output should have gone to client
# immediately via the logger.
raise ArgumentError("Failed to merge changes from %s into %s" %
(dbsource.name, dbtarget.name))
# FIXME: Run tests before pushing back to template-king.
# Use a different try/except and a specific error message.
if dryrun:
session.rollback()
return
run_git(["push", "origin", dbtarget.name],
path=temprepo,
logger=logger)
assert isinstance(OS, OperatingSystem), 'No os @ %s' % func_name()
PRSNLTY = Personality.get_unique(sess, name='generic', archetype=ARCH)
assert isinstance(PRSNLTY, Personality), 'no personality @ %s' % func_name()
NETWORK = sess.query(Network).filter(Network.cidr < 31).first()
assert isinstance(NETWORK, Network), 'no network in %s' % func_name()
DNS_DOMAIN = DnsDomain.get_unique(sess, DNAME)
assert isinstance(DNS_DOMAIN, DnsDomain), 'no dns domain @ %s' % func_name()
BRANCH = sess.query(Branch).first()
if not BRANCH:
BRANCH = Branch(branch_type='domain',
name='ny-prod',
is_sync_valid=1,
compiler='/ms/dist/elfms/PROJ/panc/prod/lib/panc.jar',
autosync=1,
owner_id=1)
add(sess, BRANCH)
commit(sess)
print BRANCH
def teardown():
#print '%s.teardown()' % func_name()
a_records = sess.query(FutureARecord).filter(
FutureARecord.name.like(SHORT_NAME_PREFIX + '%')).all()
for rec in a_records:
# this means that deleting the PNA deletes the host table.
