def getChangeQueue(self, change, existing=None):
# creates a new change queue for every project-ref
# combination.
if existing:
return DynamicChangeQueueContextManager(existing)
# Don't use Pipeline.getQueue to find an existing queue
# because we're matching project and (branch or ref).
for queue in self.pipeline.queues:
if (queue.queue[-1].change.project == change.project
and ((hasattr(change, 'branch')
and hasattr(queue.queue[-1].change, 'branch')
and queue.queue[-1].change.branch == change.branch)
or queue.queue[-1].change.ref == change.ref)):
self.log.debug("Found existing queue %s", queue)
return DynamicChangeQueueContextManager(queue)
change_queue = model.ChangeQueue(self.pipeline,
window=1,
window_floor=1,
window_increase_type='none',
window_decrease_type='none')
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def handle_project_freeze_jobs(self, gear_job):
args = json.loads(gear_job.arguments)
tenant = self.sched.abide.tenants.get(args.get("tenant"))
project = None
pipeline = None
if tenant:
(trusted, project) = tenant.getProject(args.get("project"))
pipeline = tenant.layout.pipelines.get(args.get("pipeline"))
if not project or not pipeline:
gear_job.sendWorkComplete(json.dumps(None))
return
change = model.Branch(project)
change.branch = args.get("branch", "master")
queue = model.ChangeQueue(pipeline)
item = model.QueueItem(queue, change, None)
item.layout = tenant.layout
item.freezeJobGraph(skip_file_matcher=True)
output = []
for job in item.job_graph.getJobs():
job.setBase(tenant.layout)
output.append({
'name':
job.name,
'dependencies':
list(map(lambda x: x.toDict(), job.dependencies)),
})
gear_job.sendWorkComplete(json.dumps(output))
def setUp(self):
super(TestJob, self).setUp()
self.connections = zuul.lib.connections.ConnectionRegistry()
self.addCleanup(self.connections.stop)
self.connection = Dummy(connection_name='dummy_connection')
self.source = Dummy(canonical_hostname='git.example.com',
connection=self.connection)
self.tenant = model.Tenant('tenant')
self.tenant.default_ansible_version = AnsibleManager().default_version
self.layout = model.Layout(self.tenant)
self.project = model.Project('project', self.source)
self.context = model.SourceContext(self.project, 'master', 'test',
True)
self.untrusted_context = model.SourceContext(self.project, 'master',
'test', False)
self.tpc = model.TenantProjectConfig(self.project)
self.tenant.addUntrustedProject(self.tpc)
self.pipeline = model.Pipeline('gate', self.tenant)
self.pipeline.source_context = self.context
self.layout.addPipeline(self.pipeline)
self.queue = model.ChangeQueue(self.pipeline)
self.pcontext = configloader.ParseContext(self.connections,
None, self.tenant,
AnsibleManager())
private_key_file = os.path.join(FIXTURE_DIR, 'private.pem')
with open(private_key_file, "rb") as f:
priv, pub = encryption.deserialize_rsa_keypair(f.read())
self.project.private_secrets_key = priv
self.project.public_secrets_key = pub
m = yaml.Mark('name', 0, 0, 0, '', 0)
self.start_mark = configloader.ZuulMark(m, m, '')
def constructChangeQueue(self, queue_name):
return model.ChangeQueue(self.pipeline,
window=1,
window_floor=1,
window_increase_type='none',
window_decrease_type='none',
name=queue_name)
def getChangeQueue(self, change, existing=None):
# creates a new change queue for every change
if existing:
return DynamicChangeQueueContextManager(existing)
change_queue = model.ChangeQueue(self.pipeline)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def getChangeQueue(self, change, existing=None):
# We ignore any shared change queues on the pipeline and
# instead create a new change queue for every change.
if existing:
return DynamicChangeQueueContextManager(existing)
change_queue = model.ChangeQueue(self.pipeline)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def test_inheritance_keeps_matchers(self):
pipeline = model.Pipeline('gate', self.layout)
self.layout.addPipeline(pipeline)
queue = model.ChangeQueue(pipeline)
base = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
'parent': None,
'timeout': 30,
})
self.layout.addJob(base)
python27 = self.pcontext.job_parser.fromYaml({
'_source_context':
self.context,
'_start_mark':
self.start_mark,
'name':
'python27',
'parent':
'base',
'timeout':
40,
'irrelevant-files': ['^ignored-file$'],
})
self.layout.addJob(python27)
project_config = self.pcontext.project_parser.fromYaml([{
'_source_context':
self.context,
'_start_mark':
self.start_mark,
'name':
'project',
'gate': {
'jobs': [
'python27',
]
}
}])
self.layout.addProjectConfig(project_config)
change = model.Change(self.project)
change.branch = 'master'
change.files = ['/COMMIT_MSG', 'ignored-file']
item = queue.enqueueChange(change)
item.layout = self.layout
self.assertTrue(base.changeMatches(change))
self.assertFalse(python27.changeMatches(change))
item.freezeJobGraph()
self.assertEqual([], item.getJobs())
def getChangeQueue(self, change, existing=None):
if existing:
return StaticChangeQueueContextManager(existing)
queue = self.pipeline.getQueue(change.project)
if queue:
return StaticChangeQueueContextManager(queue)
else:
# There is no existing queue for this change. Create a
# dynamic one for this one change's use
change_queue = model.ChangeQueue(self.pipeline, dynamic=True)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
self.log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def getChangeQueue(self, change, event, existing=None):
log = get_annotated_logger(self.log, event)
# Ignore the existing queue, since we can always get the correct queue
# from the pipeline. This avoids enqueuing changes in a wrong queue
# e.g. during re-configuration.
queue = self.pipeline.getQueue(change.project)
if queue:
return StaticChangeQueueContextManager(queue)
else:
# There is no existing queue for this change. Create a
# dynamic one for this one change's use
change_queue = model.ChangeQueue(self.pipeline, dynamic=True)
change_queue.addProject(change.project)
self.pipeline.addQueue(change_queue)
log.debug("Dynamically created queue %s", change_queue)
return DynamicChangeQueueContextManager(change_queue)
def buildChangeQueues(self, layout):
self.log.debug("Building shared change queues")
change_queues = {}
tenant = self.pipeline.tenant
layout_project_configs = layout.project_configs
for project_name, project_configs in layout_project_configs.items():
(trusted, project) = tenant.getProject(project_name)
queue_name = None
project_in_pipeline = False
for project_config in layout.getAllProjectConfigs(project_name):
project_pipeline_config = project_config.pipelines.get(
self.pipeline.name)
if project_pipeline_config is None:
continue
project_in_pipeline = True
queue_name = project_pipeline_config.queue_name
if queue_name:
break
if not project_in_pipeline:
continue
if queue_name and queue_name in change_queues:
change_queue = change_queues[queue_name]
else:
p = self.pipeline
change_queue = model.ChangeQueue(
p,
window=p.window,
window_floor=p.window_floor,
window_increase_type=p.window_increase_type,
window_increase_factor=p.window_increase_factor,
window_decrease_type=p.window_decrease_type,
window_decrease_factor=p.window_decrease_factor,
name=queue_name)
if queue_name:
# If this is a named queue, keep track of it in
# case it is referenced again. Otherwise, it will
# have a name automatically generated from its
# constituent projects.
change_queues[queue_name] = change_queue
self.pipeline.addQueue(change_queue)
self.log.debug("Created queue: %s" % change_queue)
change_queue.addProject(project)
self.log.debug("Added project %s to queue: %s" %
(project, change_queue))
def test_job_inheritance_job_tree(self):
queue = model.ChangeQueue(self.pipeline)
base = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'base',
'parent': None,
'timeout': 30,
})
self.layout.addJob(base)
python27 = self.pcontext.job_parser.fromYaml({
'_source_context': self.context,
'_start_mark': self.start_mark,
'name': 'python27',
'parent': 'base',
'timeout': 40,
})
self.layout.addJob(python27)
python27diablo = self.pcontext.job_parser.fromYaml({
'_source_context':
self.context,
'_start_mark':
self.start_mark,
'name':
'python27',
'branches': ['stable/diablo'],
'timeout':
50,
})
self.layout.addJob(python27diablo)
project_config = self.pcontext.project_parser.fromYaml({
'_source_context':
self.context,
'_start_mark':
self.start_mark,
'name':
'project',
'gate': {
'jobs': [{
'python27': {
'timeout': 70,
'run': 'playbooks/python27.yaml'
}
}]
}
})
self.layout.addProjectConfig(project_config)
change = model.Change(self.project)
change.branch = 'master'
item = queue.enqueueChange(change, None)
item.layout = self.layout
self.assertTrue(base.changeMatchesBranch(change))
self.assertTrue(python27.changeMatchesBranch(change))
self.assertFalse(python27diablo.changeMatchesBranch(change))
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
self.assertEqual(job.timeout, 70)
change.branch = 'stable/diablo'
item = queue.enqueueChange(change, None)
item.layout = self.layout
self.assertTrue(base.changeMatchesBranch(change))
self.assertTrue(python27.changeMatchesBranch(change))
self.assertTrue(python27diablo.changeMatchesBranch(change))
item.freezeJobGraph()
self.assertEqual(len(item.getJobs()), 1)
job = item.getJobs()[0]
self.assertEqual(job.name, 'python27')
self.assertEqual(job.timeout, 70)
