def getChangeQueue(self, change, existing=None): # creates a new change queue for every project-ref # combination. if existing: return DynamicChangeQueueContextManager(existing) # Don't use Pipeline.getQueue to find an existing queue # because we're matching project and (branch or ref). for queue in self.pipeline.queues: if (queue.queue[-1].change.project == change.project and ((hasattr(change, 'branch') and hasattr(queue.queue[-1].change, 'branch') and queue.queue[-1].change.branch == change.branch) or queue.queue[-1].change.ref == change.ref)): self.log.debug("Found existing queue %s", queue) return DynamicChangeQueueContextManager(queue) change_queue = model.ChangeQueue(self.pipeline, window=1, window_floor=1, window_increase_type='none', window_decrease_type='none') change_queue.addProject(change.project) self.pipeline.addQueue(change_queue) self.log.debug("Dynamically created queue %s", change_queue) return DynamicChangeQueueContextManager(change_queue)
def handle_project_freeze_jobs(self, gear_job): args = json.loads(gear_job.arguments) tenant = self.sched.abide.tenants.get(args.get("tenant")) project = None pipeline = None if tenant: (trusted, project) = tenant.getProject(args.get("project")) pipeline = tenant.layout.pipelines.get(args.get("pipeline")) if not project or not pipeline: gear_job.sendWorkComplete(json.dumps(None)) return change = model.Branch(project) change.branch = args.get("branch", "master") queue = model.ChangeQueue(pipeline) item = model.QueueItem(queue, change, None) item.layout = tenant.layout item.freezeJobGraph(skip_file_matcher=True) output = [] for job in item.job_graph.getJobs(): job.setBase(tenant.layout) output.append({ 'name': job.name, 'dependencies': list(map(lambda x: x.toDict(), job.dependencies)), }) gear_job.sendWorkComplete(json.dumps(output))
def setUp(self): super(TestJob, self).setUp() self.connections = zuul.lib.connections.ConnectionRegistry() self.addCleanup(self.connections.stop) self.connection = Dummy(connection_name='dummy_connection') self.source = Dummy(canonical_hostname='git.example.com', connection=self.connection) self.tenant = model.Tenant('tenant') self.tenant.default_ansible_version = AnsibleManager().default_version self.layout = model.Layout(self.tenant) self.project = model.Project('project', self.source) self.context = model.SourceContext(self.project, 'master', 'test', True) self.untrusted_context = model.SourceContext(self.project, 'master', 'test', False) self.tpc = model.TenantProjectConfig(self.project) self.tenant.addUntrustedProject(self.tpc) self.pipeline = model.Pipeline('gate', self.tenant) self.pipeline.source_context = self.context self.layout.addPipeline(self.pipeline) self.queue = model.ChangeQueue(self.pipeline) self.pcontext = configloader.ParseContext(self.connections, None, self.tenant, AnsibleManager()) private_key_file = os.path.join(FIXTURE_DIR, 'private.pem') with open(private_key_file, "rb") as f: priv, pub = encryption.deserialize_rsa_keypair(f.read()) self.project.private_secrets_key = priv self.project.public_secrets_key = pub m = yaml.Mark('name', 0, 0, 0, '', 0) self.start_mark = configloader.ZuulMark(m, m, '')
def constructChangeQueue(self, queue_name): return model.ChangeQueue(self.pipeline, window=1, window_floor=1, window_increase_type='none', window_decrease_type='none', name=queue_name)
def getChangeQueue(self, change, existing=None): # creates a new change queue for every change if existing: return DynamicChangeQueueContextManager(existing) change_queue = model.ChangeQueue(self.pipeline) change_queue.addProject(change.project) self.pipeline.addQueue(change_queue) self.log.debug("Dynamically created queue %s", change_queue) return DynamicChangeQueueContextManager(change_queue)
def getChangeQueue(self, change, existing=None): # We ignore any shared change queues on the pipeline and # instead create a new change queue for every change. if existing: return DynamicChangeQueueContextManager(existing) change_queue = model.ChangeQueue(self.pipeline) change_queue.addProject(change.project) self.pipeline.addQueue(change_queue) self.log.debug("Dynamically created queue %s", change_queue) return DynamicChangeQueueContextManager(change_queue)
def test_inheritance_keeps_matchers(self): pipeline = model.Pipeline('gate', self.layout) self.layout.addPipeline(pipeline) queue = model.ChangeQueue(pipeline) base = self.pcontext.job_parser.fromYaml({ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'base', 'parent': None, 'timeout': 30, }) self.layout.addJob(base) python27 = self.pcontext.job_parser.fromYaml({ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'python27', 'parent': 'base', 'timeout': 40, 'irrelevant-files': ['^ignored-file$'], }) self.layout.addJob(python27) project_config = self.pcontext.project_parser.fromYaml([{ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'project', 'gate': { 'jobs': [ 'python27', ] } }]) self.layout.addProjectConfig(project_config) change = model.Change(self.project) change.branch = 'master' change.files = ['/COMMIT_MSG', 'ignored-file'] item = queue.enqueueChange(change) item.layout = self.layout self.assertTrue(base.changeMatches(change)) self.assertFalse(python27.changeMatches(change)) item.freezeJobGraph() self.assertEqual([], item.getJobs())
def getChangeQueue(self, change, existing=None): if existing: return StaticChangeQueueContextManager(existing) queue = self.pipeline.getQueue(change.project) if queue: return StaticChangeQueueContextManager(queue) else: # There is no existing queue for this change. Create a # dynamic one for this one change's use change_queue = model.ChangeQueue(self.pipeline, dynamic=True) change_queue.addProject(change.project) self.pipeline.addQueue(change_queue) self.log.debug("Dynamically created queue %s", change_queue) return DynamicChangeQueueContextManager(change_queue)
def getChangeQueue(self, change, event, existing=None): log = get_annotated_logger(self.log, event) # Ignore the existing queue, since we can always get the correct queue # from the pipeline. This avoids enqueuing changes in a wrong queue # e.g. during re-configuration. queue = self.pipeline.getQueue(change.project) if queue: return StaticChangeQueueContextManager(queue) else: # There is no existing queue for this change. Create a # dynamic one for this one change's use change_queue = model.ChangeQueue(self.pipeline, dynamic=True) change_queue.addProject(change.project) self.pipeline.addQueue(change_queue) log.debug("Dynamically created queue %s", change_queue) return DynamicChangeQueueContextManager(change_queue)
def buildChangeQueues(self, layout): self.log.debug("Building shared change queues") change_queues = {} tenant = self.pipeline.tenant layout_project_configs = layout.project_configs for project_name, project_configs in layout_project_configs.items(): (trusted, project) = tenant.getProject(project_name) queue_name = None project_in_pipeline = False for project_config in layout.getAllProjectConfigs(project_name): project_pipeline_config = project_config.pipelines.get( self.pipeline.name) if project_pipeline_config is None: continue project_in_pipeline = True queue_name = project_pipeline_config.queue_name if queue_name: break if not project_in_pipeline: continue if queue_name and queue_name in change_queues: change_queue = change_queues[queue_name] else: p = self.pipeline change_queue = model.ChangeQueue( p, window=p.window, window_floor=p.window_floor, window_increase_type=p.window_increase_type, window_increase_factor=p.window_increase_factor, window_decrease_type=p.window_decrease_type, window_decrease_factor=p.window_decrease_factor, name=queue_name) if queue_name: # If this is a named queue, keep track of it in # case it is referenced again. Otherwise, it will # have a name automatically generated from its # constituent projects. change_queues[queue_name] = change_queue self.pipeline.addQueue(change_queue) self.log.debug("Created queue: %s" % change_queue) change_queue.addProject(project) self.log.debug("Added project %s to queue: %s" % (project, change_queue))
def test_job_inheritance_job_tree(self): queue = model.ChangeQueue(self.pipeline) base = self.pcontext.job_parser.fromYaml({ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'base', 'parent': None, 'timeout': 30, }) self.layout.addJob(base) python27 = self.pcontext.job_parser.fromYaml({ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'python27', 'parent': 'base', 'timeout': 40, }) self.layout.addJob(python27) python27diablo = self.pcontext.job_parser.fromYaml({ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'python27', 'branches': ['stable/diablo'], 'timeout': 50, }) self.layout.addJob(python27diablo) project_config = self.pcontext.project_parser.fromYaml({ '_source_context': self.context, '_start_mark': self.start_mark, 'name': 'project', 'gate': { 'jobs': [{ 'python27': { 'timeout': 70, 'run': 'playbooks/python27.yaml' } }] } }) self.layout.addProjectConfig(project_config) change = model.Change(self.project) change.branch = 'master' item = queue.enqueueChange(change, None) item.layout = self.layout self.assertTrue(base.changeMatchesBranch(change)) self.assertTrue(python27.changeMatchesBranch(change)) self.assertFalse(python27diablo.changeMatchesBranch(change)) item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') self.assertEqual(job.timeout, 70) change.branch = 'stable/diablo' item = queue.enqueueChange(change, None) item.layout = self.layout self.assertTrue(base.changeMatchesBranch(change)) self.assertTrue(python27.changeMatchesBranch(change)) self.assertTrue(python27diablo.changeMatchesBranch(change)) item.freezeJobGraph() self.assertEqual(len(item.getJobs()), 1) job = item.getJobs()[0] self.assertEqual(job.name, 'python27') self.assertEqual(job.timeout, 70)