def render(self, session, logger, dbuser, sandbox, start, get, comments, **arguments): if not dbuser: raise AuthorizationException("Cannot create a sandbox without an " "authenticated connection.") sandbox = self.force_my_sandbox(session, logger, dbuser, sandbox) # See `git check-ref-format --help` for naming restrictions. # We want to layer a few extra restrictions on top of that... validate_template_name("--sandbox", sandbox) Branch.get_unique(session, sandbox, preclude=True) # Check that the user has cleared up a directory of the same # name; if this is not the case the branch may be created (in git) # and added to the database - however CommandGet will fail roleing # back the database leaving the branch created in git templatesdir = self.config.get("broker", "templatesdir") sandboxdir = os.path.join(templatesdir, dbuser.name, sandbox) if os.path.exists(sandboxdir): raise ArgumentError("Sandbox directory %s already exists; " "cannot create branch." % sandboxdir) if not start: start = self.config.get("broker", "default_domain_start") dbstart = Branch.get_unique(session, start, compel=True) kingdir = self.config.get("broker", "kingdir") base_commit = run_git(["show-ref", "--hash", "refs/heads/" + dbstart.name], logger=logger, path=kingdir) compiler = self.config.get("panc", "pan_compiler") dbsandbox = Sandbox(name=sandbox, owner=dbuser, compiler=compiler, base_commit=base_commit, comments=comments) session.add(dbsandbox) session.flush() # Currently this will fail if the branch already exists... # That seems like the right behavior. It's an internal # consistency issue that would need to be addressed explicitly. run_git(["branch", sandbox, dbstart.name], logger=logger, path=kingdir) # If we arrive there the above "git branch" command has succeeded; # therefore we should comit the changes to the database. If this is # not done, and CommandGet fails (see dir check above), then the # git branch will be created but the database changes roled back. session.commit() if get == False: # The client knows to interpret an empty response as no action. return [] return CommandGet.render(self, session=session, logger=logger, dbuser=dbuser, sandbox=sandbox)
def test_create_hosts(): br = Branch.get_unique(sess, 'ny-prod', compel=True) dns_dmn = DnsDomain.get_unique(sess, 'one-nyp.ms.com', compel=True) stat = Status.get_unique(sess, 'build', compel=True) os = sess.query(OperatingSystem).filter(Archetype.name == 'vmhost').first() assert os, 'No OS in %s' % func_name() pers = sess.query(Personality).select_from(join( Personality, Archetype)).filter( and_(Archetype.name == 'vmhost', Personality.name == 'generic')).one() sess.autoflush = False for i in xrange(NUM_HOSTS): machine = m_factory.next() vm_host = Host(machine=machine, name='%s%s' % (HOST_NAME, i), dns_domain=dns_dmn, branch=br, personality=pers, status=stat, operating_system=os) add(sess, vm_host) sess.autoflush = True commit(sess) hosts = sess.query(Host).filter(Host.name.like(HOST_NAME + '%')).all() assert len(hosts) is NUM_HOSTS print 'created %s hosts' % len(hosts)
def test_create_hosts(): br = Branch.get_unique(sess, 'ny-prod', compel=True) dns_dmn = DnsDomain.get_unique(sess, 'one-nyp.ms.com', compel=True) stat = Status.get_unique(sess, 'build', compel=True) os = sess.query(OperatingSystem).filter(Archetype.name == 'vmhost').first() assert os, 'No OS in %s' % func_name() pers = sess.query(Personality).select_from( join(Personality, Archetype)).filter( and_(Archetype.name=='vmhost', Personality.name=='generic')).one() sess.autoflush=False for i in xrange(NUM_HOSTS): machine = m_factory.next() vm_host = Host(machine=machine, name='%s%s' % (HOST_NAME, i), dns_domain=dns_dmn, branch=br, personality=pers, status=stat, operating_system=os) add(sess, vm_host) sess.autoflush=True commit(sess) hosts = sess.query(Host).filter( Host.name.like(HOST_NAME+'%')).all() assert len(hosts) is NUM_HOSTS print 'created %s hosts'% len(hosts)
def render(self, session, logger, branch, comments, **arguments): dbbranch = Branch.get_unique(session, branch, compel=True) if dbbranch.is_sync_valid: logger.warn("{0} already marked as valid.".format(dbbranch)) # For now, comments just get stored in the audit log. dbbranch.is_sync_valid = True session.flush() # FIXME: Run tests? return
def render(self, session, logger, branch, comments, **arguments): dbbranch = Branch.get_unique(session, branch, compel=True) if dbbranch.is_sync_valid: logger.warn("{0} already marked as valid.".format(dbbranch)) # For now, comments just get stored in the audit log. dbbranch.is_sync_valid = True session.add(dbbranch) # FIXME: Run tests? return
def render(self, session, logger, dbuser, sandbox, start, get, comments, **arguments): if not dbuser: raise AuthorizationException("Cannot create a sandbox without an " "authenticated connection.") sandbox = self.force_my_sandbox(session, logger, dbuser, sandbox) # See `git check-ref-format --help` for naming restrictions. # We want to layer a few extra restrictions on top of that... valid = re.compile('^[a-zA-Z0-9_.-]+$') if (not valid.match(sandbox)): raise ArgumentError("sandbox name '%s' is not valid" % sandbox) Branch.get_unique(session, sandbox, preclude=True) if not start: start = self.config.get("broker", "default_domain_start") dbstart = Branch.get_unique(session, start, compel=True) kingdir = self.config.get("broker", "kingdir") base_commit = run_git(["show-ref", "--hash", "refs/heads/" + dbstart.name], logger=logger, path=kingdir) compiler = self.config.get("panc", "pan_compiler") dbsandbox = Sandbox(name=sandbox, owner=dbuser, compiler=compiler, base_commit=base_commit, comments=comments) session.add(dbsandbox) session.flush() # Currently this will fail if the branch already exists... # That seems like the right behavior. It's an internal # consistency issue that would need to be addressed explicitly. run_git(["branch", sandbox, dbstart.name], logger=logger, path=kingdir) if get == False: # The client knows to interpret an empty response as no action. return [] return CommandGet.render(self, session=session, logger=logger, dbuser=dbuser, sandbox=sandbox)
def test_no_host_threshold(): """ ensure down_hosts_threshold must exist """ br = Branch.get_unique(sess, 'ny-prod', compel=True) np = Building.get_unique(sess, name='np', compel=True) per = sess.query(Personality).select_from( join(Archetype, Personality)).filter( and_(Archetype.name=='windows', Personality.name=='generic')).one() ec = EsxCluster(name=CLUSTER_NAME, location_constraint=np, personality=per, branch=br) add(sess, ec) commit(sess)
def test_no_host_threshold(): """ ensure down_hosts_threshold must exist """ br = Branch.get_unique(sess, 'ny-prod', compel=True) np = Building.get_unique(sess, name='np', compel=True) per = sess.query(Personality).select_from(join( Archetype, Personality)).filter( and_(Archetype.name == 'windows', Personality.name == 'generic')).one() ec = EsxCluster(name=CLUSTER_NAME, location_constraint=np, personality=per, branch=br) add(sess, ec) commit(sess)
def render(self, session, track, not_tracking, change_manager, fullinfo, **arguments): q = search_branch_query(self.config, session, Domain, **arguments) if track: dbtracked = Branch.get_unique(session, track, compel=True) q = q.filter_by(tracked_branch=dbtracked) if not_tracking: q = q.filter_by(tracked_branch=None) if change_manager is not None: q = q.filter_by(requires_change_manager=change_manager) q = q.order_by(Domain.name) result = q.all() if fullinfo: return result else: return StringList(result)
def test_create_esx_cluster(): """ tests the creation of an EsxCluster """ np = Building.get_unique(sess, name='np', compel=True) br = Branch.get_unique(sess, 'ny-prod', compel=True) per = sess.query(Personality).select_from( join(Archetype, Personality)).filter( and_(Archetype.name=='windows', Personality.name=='generic')).one() ec = EsxCluster(name=CLUSTER_NAME, location_constraint=np, personality=per, down_hosts_threshold=2, branch=br) create(sess, ec) assert ec print ec assert ec.max_hosts is 8 print 'esx cluster max members = %s' % ec.max_hosts
def get_branch_and_author(session, logger, domain=None, sandbox=None, branch=None, compel=False): dbbranch = None dbauthor = None if domain: dbbranch = Domain.get_unique(session, domain, compel=True) elif branch: dbbranch = Branch.get_unique(session, branch, compel=True) elif sandbox: (author, slash, name) = sandbox.partition('/') if not slash: raise ArgumentError("Expected sandbox as 'author/branch', author " "name and branch name separated by a slash.") dbbranch = Sandbox.get_unique(session, name, compel=True) dbauthor = get_user_principal(session, author) elif compel: raise ArgumentError("Please specify either sandbox or domain.") return (dbbranch, dbauthor)
def test_create_esx_cluster(): """ tests the creation of an EsxCluster """ np = Building.get_unique(sess, name='np', compel=True) br = Branch.get_unique(sess, 'ny-prod', compel=True) per = sess.query(Personality).select_from(join( Archetype, Personality)).filter( and_(Archetype.name == 'windows', Personality.name == 'generic')).one() ec = EsxCluster(name=CLUSTER_NAME, location_constraint=np, personality=per, down_hosts_threshold=2, branch=br) create(sess, ec) assert ec print ec assert ec.max_hosts is 8 print 'esx cluster max members = %s' % ec.max_hosts
def test_create_clusters(): np = sess.query(Building).filter_by(name='np').one() br = Branch.get_unique(sess, 'ny-prod', compel=True) per = sess.query(Personality).select_from( join(Archetype, Personality)).filter( and_(Archetype.name == 'windows', Personality.name == 'generic')).one() for i in xrange(NUM_CLUSTERS): ec = EsxCluster(name='%s%s' % (CLUSTER_NAME, i), location_constraint=np, branch=br, personality=per, down_hosts_threshold=2) add(sess, ec) commit(sess) ecs = sess.query(EsxCluster).all() assert len(ecs) is NUM_CLUSTERS print ecs[0] assert ecs[0].max_hosts is 8 print 'esx cluster max hosts = %s' % (ecs[0].max_hosts)
def render(self, session, logger, source, target, sync, dryrun, comments, justification, user, requestid, **arguments): # Most of the logic here is duplicated in publish dbsource = Branch.get_unique(session, source, compel=True) # The target has to be a non-tracking domain dbtarget = Domain.get_unique(session, target, compel=True) if sync and isinstance(dbtarget.tracked_branch, Domain) \ and dbtarget.tracked_branch.autosync and dbtarget.autosync: # The user probably meant to deploy to the tracked branch, # but only do so if all the relevant autosync flags are # positive. logger.warning("Deploying to tracked branch %s and then will " "auto-sync %s" % ( dbtarget.tracked_branch.name, dbtarget.name)) dbtarget = dbtarget.tracked_branch elif dbtarget.tracked_branch: raise ArgumentError("Cannot deploy to tracking domain %s. " "Did you mean domain %s?" % (dbtarget.name, dbtarget.tracked_branch.name)) if sync and not dbtarget.is_sync_valid and dbtarget.trackers: # FIXME: Maybe raise an ArgumentError and request that the # command run with --nosync? Maybe provide a --validate flag? # For now, just auto-flip (below). pass if not dbtarget.is_sync_valid: dbtarget.is_sync_valid = True if dbtarget.requires_change_manager: if not justification: raise AuthorizationException( "{0} is under change management control. Please specify " "--justification.".format(dbtarget)) validate_justification(user, justification) if isinstance(dbsource, Sandbox): domainsdir = self.config.get('broker', 'domainsdir') targetdir = os.path.join(domainsdir, dbtarget.name) filterre = re.compile('^' + dbsource.base_commit + '$') found = run_git(['rev-list', 'HEAD'], path=targetdir, logger=logger, filterre=filterre) if not found: raise ArgumentError("You're trying to deploy a sandbox to a " "domain that does not contain the commit " "where the sandbox was branched from.") kingdir = self.config.get("broker", "kingdir") rundir = self.config.get("broker", "rundir") tempdir = mkdtemp(prefix="deploy_", suffix="_%s" % dbsource.name, dir=rundir) try: run_git(["clone", "--shared", "--branch", dbtarget.name, kingdir, dbtarget.name], path=tempdir, logger=logger) temprepo = os.path.join(tempdir, dbtarget.name) # We could try to use fmt-merge-msg but its usage is so obscure that # faking it is easier merge_msg = [] merge_msg.append("Merge remote branch 'origin/%s' into %s" % (dbsource.name, dbtarget.name)) merge_msg.append("") merge_msg.append("User: %s" % user) merge_msg.append("Request ID: %s" % requestid) if justification: merge_msg.append("Justification: %s" % justification) if comments: merge_msg.append("Comments: %s" % comments) try: run_git(["merge", "--no-ff", "origin/%s" % dbsource.name, "-m", "\n".join(merge_msg)], path=temprepo, logger=logger, loglevel=CLIENT_INFO) except ProcessException, e: # No need to re-print e, output should have gone to client # immediately via the logger. raise ArgumentError("Failed to merge changes from %s into %s" % (dbsource.name, dbtarget.name)) # FIXME: Run tests before pushing back to template-king. # Use a different try/except and a specific error message. if dryrun: session.rollback() return run_git(["push", "origin", dbtarget.name], path=temprepo, logger=logger)
def render(self, session, logger, dbuser, domain, track, start, change_manager, comments, allow_manage, **arguments): if not dbuser: raise AuthorizationException("Cannot create a domain without " "an authenticated connection.") Branch.get_unique(session, domain, preclude=True) valid = re.compile('^[a-zA-Z0-9_.-]+$') if (not valid.match(domain)): raise ArgumentError("Domain name '%s' is not valid." % domain) # FIXME: Verify that track is a valid branch name? # Or just let the branch command fail? compiler = self.config.get("panc", "pan_compiler") dbtracked = None if track: dbtracked = Branch.get_unique(session, track, compel=True) if getattr(dbtracked, "tracked_branch", None): raise ArgumentError("Cannot nest tracking. Try tracking " "{0:l} directly.".format(dbtracked.tracked_branch)) start_point = dbtracked if change_manager: raise ArgumentError("Cannot enforce a change manager for " "tracking domains.") else: if not start: start = self.config.get("broker", "default_domain_start") start_point = Branch.get_unique(session, start, compel=True) dbdomain = Domain(name=domain, owner=dbuser, compiler=compiler, tracked_branch=dbtracked, requires_change_manager=bool(change_manager), comments=comments) session.add(dbdomain) if allow_manage is not None: dbdomain.allow_manage = allow_manage session.flush() domainsdir = self.config.get("broker", "domainsdir") clonedir = os.path.join(domainsdir, dbdomain.name) if os.path.exists(clonedir): raise InternalError("Domain directory already exists") kingdir = self.config.get("broker", "kingdir") cmd = ["branch"] if track: cmd.append("--track") else: cmd.append("--no-track") cmd.append(dbdomain.name) cmd.append(start_point.name) run_git(cmd, path=kingdir, logger=logger) # If the branch command above fails the DB will roll back as normal. # If the command below fails we need to clean up from itself and above. try: run_git(["clone", "--branch", dbdomain.name, kingdir, dbdomain.name], path=domainsdir, logger=logger) except ProcessException, e: try: remove_dir(clonedir, logger=logger) run_git(["branch", "-D", dbdomain.name], path=kingdir, logger=logger) except ProcessException, e2: logger.info("Exception while cleaning up: %s", e2)
def render(self, session, logger, dbuser, domain, track, start, change_manager, comments, allow_manage, **arguments): if not dbuser: raise AuthorizationException("Cannot create a domain without " "an authenticated connection.") Branch.get_unique(session, domain, preclude=True) valid = re.compile('^[a-zA-Z0-9_.-]+$') if (not valid.match(domain)): raise ArgumentError("Domain name '%s' is not valid." % domain) # FIXME: Verify that track is a valid branch name? # Or just let the branch command fail? compiler = self.config.get("panc", "pan_compiler") dbtracked = None if track: dbtracked = Branch.get_unique(session, track, compel=True) if getattr(dbtracked, "tracked_branch", None): raise ArgumentError("Cannot nest tracking. Try tracking " "{0:l} directly.".format( dbtracked.tracked_branch)) start_point = dbtracked if change_manager: raise ArgumentError("Cannot enforce a change manager for " "tracking domains.") else: if not start: start = self.config.get("broker", "default_domain_start") start_point = Branch.get_unique(session, start, compel=True) dbdomain = Domain(name=domain, owner=dbuser, compiler=compiler, tracked_branch=dbtracked, requires_change_manager=bool(change_manager), comments=comments) session.add(dbdomain) if allow_manage is not None: dbdomain.allow_manage = allow_manage session.flush() domainsdir = self.config.get("broker", "domainsdir") clonedir = os.path.join(domainsdir, dbdomain.name) if os.path.exists(clonedir): raise InternalError("Domain directory already exists") kingdir = self.config.get("broker", "kingdir") cmd = ["branch"] if track: cmd.append("--track") else: cmd.append("--no-track") cmd.append(dbdomain.name) cmd.append(start_point.name) run_git(cmd, path=kingdir, logger=logger) # If the branch command above fails the DB will roll back as normal. # If the command below fails we need to clean up from itself and above. try: run_git( ["clone", "--branch", dbdomain.name, kingdir, dbdomain.name], path=domainsdir, logger=logger) except ProcessException, e: try: remove_dir(clonedir, logger=logger) run_git(["branch", "-D", dbdomain.name], path=kingdir, logger=logger) except ProcessException, e2: logger.info("Exception while cleaning up: %s", e2)
def render(self, session, logger, source, target, sync, dryrun, comments, justification, user, requestid, **arguments): # Most of the logic here is duplicated in publish dbsource = Branch.get_unique(session, source, compel=True) # The target has to be a non-tracking domain dbtarget = Domain.get_unique(session, target, compel=True) if sync and isinstance(dbtarget.tracked_branch, Domain) \ and dbtarget.tracked_branch.autosync and dbtarget.autosync: # The user probably meant to deploy to the tracked branch, # but only do so if all the relevant autosync flags are # positive. logger.warning("Deploying to tracked branch %s and then will " "auto-sync %s" % (dbtarget.tracked_branch.name, dbtarget.name)) dbtarget = dbtarget.tracked_branch elif dbtarget.tracked_branch: raise ArgumentError("Cannot deploy to tracking domain %s. " "Did you mean domain %s?" % (dbtarget.name, dbtarget.tracked_branch.name)) if sync and not dbtarget.is_sync_valid and dbtarget.trackers: # FIXME: Maybe raise an ArgumentError and request that the # command run with --nosync? Maybe provide a --validate flag? # For now, just auto-flip (below). pass if not dbtarget.is_sync_valid: dbtarget.is_sync_valid = True if dbtarget.requires_change_manager: if not justification: raise AuthorizationException( "{0} is under change management control. Please specify " "--justification.".format(dbtarget)) validate_justification(user, justification) if isinstance(dbsource, Sandbox): domainsdir = self.config.get('broker', 'domainsdir') targetdir = os.path.join(domainsdir, dbtarget.name) filterre = re.compile('^' + dbsource.base_commit + '$') found = run_git(['rev-list', 'HEAD'], path=targetdir, logger=logger, filterre=filterre) if not found: raise ArgumentError("You're trying to deploy a sandbox to a " "domain that does not contain the commit " "where the sandbox was branched from.") kingdir = self.config.get("broker", "kingdir") rundir = self.config.get("broker", "rundir") tempdir = mkdtemp(prefix="deploy_", suffix="_%s" % dbsource.name, dir=rundir) try: run_git([ "clone", "--shared", "--branch", dbtarget.name, kingdir, dbtarget.name ], path=tempdir, logger=logger) temprepo = os.path.join(tempdir, dbtarget.name) # We could try to use fmt-merge-msg but its usage is so obscure that # faking it is easier merge_msg = [] merge_msg.append("Merge remote branch 'origin/%s' into %s" % (dbsource.name, dbtarget.name)) merge_msg.append("") merge_msg.append("User: %s" % user) merge_msg.append("Request ID: %s" % requestid) if justification: merge_msg.append("Justification: %s" % justification) if comments: merge_msg.append("Comments: %s" % comments) try: run_git([ "merge", "--no-ff", "origin/%s" % dbsource.name, "-m", "\n".join(merge_msg) ], path=temprepo, logger=logger, loglevel=CLIENT_INFO) except ProcessException, e: # No need to re-print e, output should have gone to client # immediately via the logger. raise ArgumentError("Failed to merge changes from %s into %s" % (dbsource.name, dbtarget.name)) # FIXME: Run tests before pushing back to template-king. # Use a different try/except and a specific error message. if dryrun: session.rollback() return run_git(["push", "origin", dbtarget.name], path=temprepo, logger=logger)
assert isinstance(OS, OperatingSystem), 'No os @ %s' % func_name() PRSNLTY = Personality.get_unique(sess, name='generic', archetype=ARCH) assert isinstance(PRSNLTY, Personality), 'no personality @ %s' % func_name() NETWORK = sess.query(Network).filter(Network.cidr < 31).first() assert isinstance(NETWORK, Network), 'no network in %s' % func_name() DNS_DOMAIN = DnsDomain.get_unique(sess, DNAME) assert isinstance(DNS_DOMAIN, DnsDomain), 'no dns domain @ %s' % func_name() BRANCH = sess.query(Branch).first() if not BRANCH: BRANCH = Branch(branch_type='domain', name='ny-prod', is_sync_valid=1, compiler='/ms/dist/elfms/PROJ/panc/prod/lib/panc.jar', autosync=1, owner_id=1) add(sess, BRANCH) commit(sess) print BRANCH def teardown(): #print '%s.teardown()' % func_name() a_records = sess.query(FutureARecord).filter( FutureARecord.name.like(SHORT_NAME_PREFIX + '%')).all() for rec in a_records: # this means that deleting the PNA deletes the host table.