def resumeBuild(self, build):
if not build.__gearman_worker:
self.log.error("Build %s has no manager while resuming" %
(build,))
resume_uuid = str(uuid4().hex)
data = dict(uuid=build.__gearman_job.unique)
stop_job = gear.TextJob("executor:resume:%s" % build.__gearman_worker,
json_dumps(data), unique=resume_uuid)
self.meta_jobs[resume_uuid] = stop_job
self.log.debug("Submitting resume job: %s", stop_job)
self.gearman.submitJob(stop_job, precedence=gear.PRECEDENCE_HIGH,
timeout=300)
def cancelRunningBuild(self, build):
if not build.__gearman_worker:
self.log.error("Build %s has no manager while canceling" %
(build, ))
stop_uuid = str(uuid4().hex)
data = dict(uuid=build.__gearman_job.unique)
stop_job = gear.TextJob("executor:stop:%s" % build.__gearman_worker,
json_dumps(data),
unique=stop_uuid)
self.meta_jobs[stop_uuid] = stop_job
self.log.debug("Submitting stop job: %s", stop_job)
self.gearman.submitJob(stop_job,
precedence=gear.PRECEDENCE_HIGH,
timeout=300)
return True
def resumeBuild(self, build):
log = get_annotated_logger(self.log, build.zuul_event_id)
if not build.__gearman_worker:
log.error("Build %s has no manager while resuming", build)
resume_uuid = str(uuid4().hex)
data = dict(uuid=build.__gearman_job.unique,
zuul_event_id=build.zuul_event_id)
stop_job = gear.TextJob("executor:resume:%s" % build.__gearman_worker,
json_dumps(data),
unique=resume_uuid)
self.meta_jobs[resume_uuid] = stop_job
log.debug("Submitting resume job: %s", stop_job)
self.gearman.submitJob(stop_job,
precedence=gear.PRECEDENCE_HIGH,
timeout=300)
def execute(self, job, item, pipeline, dependent_changes=[],
merger_items=[]):
tenant = pipeline.tenant
uuid = str(uuid4().hex)
nodeset = item.current_build_set.getJobNodeSet(job.name)
self.log.info(
"Execute job %s (uuid: %s) on nodes %s for change %s "
"with dependent changes %s" % (
job, uuid, nodeset, item.change, dependent_changes))
project = dict(
name=item.change.project.name,
short_name=item.change.project.name.split('/')[-1],
canonical_hostname=item.change.project.canonical_hostname,
canonical_name=item.change.project.canonical_name,
src_dir=os.path.join('src', item.change.project.canonical_name),
)
zuul_params = dict(build=uuid,
buildset=item.current_build_set.uuid,
ref=item.change.ref,
pipeline=pipeline.name,
job=job.name,
voting=job.voting,
project=project,
tenant=tenant.name,
timeout=job.timeout,
jobtags=sorted(job.tags),
_inheritance_path=list(job.inheritance_path))
if job.override_checkout:
zuul_params['override_checkout'] = job.override_checkout
if hasattr(item.change, 'branch'):
zuul_params['branch'] = item.change.branch
if hasattr(item.change, 'tag'):
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
zuul_params['change'] = str(item.change.number)
if hasattr(item.change, 'url'):
zuul_params['change_url'] = item.change.url
if hasattr(item.change, 'patchset'):
zuul_params['patchset'] = str(item.change.patchset)
if (hasattr(item.change, 'oldrev') and item.change.oldrev
and item.change.oldrev != '0' * 40):
zuul_params['oldrev'] = item.change.oldrev
if (hasattr(item.change, 'newrev') and item.change.newrev
and item.change.newrev != '0' * 40):
zuul_params['newrev'] = item.change.newrev
zuul_params['projects'] = {} # Set below
zuul_params['items'] = dependent_changes
params = dict()
params['job'] = job.name
params['timeout'] = job.timeout
params['post_timeout'] = job.post_timeout
params['items'] = merger_items
params['projects'] = []
if hasattr(item.change, 'branch'):
params['branch'] = item.change.branch
else:
params['branch'] = None
params['override_branch'] = job.override_branch
params['override_checkout'] = job.override_checkout
params['repo_state'] = item.current_build_set.repo_state
def make_playbook(playbook):
d = playbook.toDict()
for role in d['roles']:
if role['type'] != 'zuul':
continue
project_metadata = item.layout.getProjectMetadata(
role['project_canonical_name'])
if project_metadata:
role['project_default_branch'] = \
project_metadata.default_branch
else:
role['project_default_branch'] = 'master'
role_trusted, role_project = item.layout.tenant.getProject(
role['project_canonical_name'])
role_connection = role_project.source.connection
role['connection'] = role_connection.connection_name
role['project'] = role_project.name
return d
if job.name != 'noop':
params['playbooks'] = [make_playbook(x) for x in job.run]
params['pre_playbooks'] = [make_playbook(x) for x in job.pre_run]
params['post_playbooks'] = [make_playbook(x) for x in job.post_run]
nodes = []
for node in nodeset.getNodes():
n = node.toDict()
n.update(dict(name=node.name, label=node.label))
nodes.append(n)
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['vars'] = job.variables
params['host_vars'] = job.host_variables
params['group_vars'] = job.group_variables
params['zuul'] = zuul_params
projects = set()
required_projects = set()
def make_project_dict(project, override_branch=None,
override_checkout=None):
project_metadata = item.layout.getProjectMetadata(
project.canonical_name)
if project_metadata:
project_default_branch = project_metadata.default_branch
else:
project_default_branch = 'master'
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
canonical_name=project.canonical_name,
override_branch=override_branch,
override_checkout=override_checkout,
default_branch=project_default_branch)
if job.required_projects:
for job_project in job.required_projects.values():
(trusted, project) = tenant.getProject(
job_project.project_name)
if project is None:
raise Exception("Unknown project %s" %
(job_project.project_name,))
params['projects'].append(
make_project_dict(project,
job_project.override_branch,
job_project.override_checkout))
projects.add(project)
required_projects.add(project)
for change in dependent_changes:
# We have to find the project this way because it may not
# be registered in the tenant (ie, a foreign project).
source = self.sched.connections.getSourceByCanonicalHostname(
change['project']['canonical_hostname'])
project = source.getProject(change['project']['name'])
if project not in projects:
params['projects'].append(make_project_dict(project))
projects.add(project)
for p in projects:
zuul_params['projects'][p.canonical_name] = (dict(
name=p.name,
short_name=p.name.split('/')[-1],
# Duplicate this into the dict too, so that iterating
# project.values() is easier for callers
canonical_name=p.canonical_name,
canonical_hostname=p.canonical_hostname,
src_dir=os.path.join('src', p.canonical_name),
required=(p in required_projects),
))
build = Build(job, uuid)
build.parameters = params
build.nodeset = nodeset
self.log.debug("Adding build %s of job %s to item %s" %
(build, job, item))
item.addBuild(build)
if job.name == 'noop':
self.sched.onBuildStarted(build)
self.sched.onBuildCompleted(build, 'SUCCESS', {})
return build
gearman_job = gear.TextJob('executor:execute', json_dumps(params),
unique=uuid)
build.__gearman_job = gearman_job
build.__gearman_worker = None
self.builds[uuid] = build
if pipeline.precedence == zuul.model.PRECEDENCE_NORMAL:
precedence = gear.PRECEDENCE_NORMAL
elif pipeline.precedence == zuul.model.PRECEDENCE_HIGH:
precedence = gear.PRECEDENCE_HIGH
elif pipeline.precedence == zuul.model.PRECEDENCE_LOW:
precedence = gear.PRECEDENCE_LOW
try:
self.gearman.submitJob(gearman_job, precedence=precedence,
timeout=300)
except Exception:
self.log.exception("Unable to submit job to Gearman")
self.onBuildCompleted(gearman_job, 'EXCEPTION')
return build
if not gearman_job.handle:
self.log.error("No job handle was received for %s after"
" 300 seconds; marking as lost." %
gearman_job)
self.onBuildCompleted(gearman_job, 'NO_HANDLE')
self.log.debug("Received handle %s for %s" % (gearman_job.handle,
build))
return build
def execute(self, job, item, pipeline, dependent_changes=[],
merger_items=[]):
log = get_annotated_logger(self.log, item.event)
tenant = pipeline.tenant
uuid = str(uuid4().hex)
nodeset = item.current_build_set.getJobNodeSet(job.name)
log.info(
"Execute job %s (uuid: %s) on nodes %s for change %s "
"with dependent changes %s",
job, uuid, nodeset, item.change, dependent_changes)
project = dict(
name=item.change.project.name,
short_name=item.change.project.name.split('/')[-1],
canonical_hostname=item.change.project.canonical_hostname,
canonical_name=item.change.project.canonical_name,
src_dir=os.path.join('src', item.change.project.canonical_name),
)
zuul_params = dict(build=uuid,
buildset=item.current_build_set.uuid,
ref=item.change.ref,
pipeline=pipeline.name,
job=job.name,
voting=job.voting,
project=project,
tenant=tenant.name,
timeout=job.timeout,
event_id=item.event.zuul_event_id,
jobtags=sorted(job.tags),
_inheritance_path=list(job.inheritance_path))
if job.artifact_data:
zuul_params['artifacts'] = job.artifact_data
if job.override_checkout:
zuul_params['override_checkout'] = job.override_checkout
if hasattr(item.change, 'branch'):
zuul_params['branch'] = item.change.branch
if hasattr(item.change, 'tag'):
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
zuul_params['change'] = str(item.change.number)
if hasattr(item.change, 'url'):
zuul_params['change_url'] = item.change.url
if hasattr(item.change, 'patchset'):
zuul_params['patchset'] = str(item.change.patchset)
if hasattr(item.change, 'message'):
zuul_params['message'] = item.change.message
if (hasattr(item.change, 'oldrev') and item.change.oldrev
and item.change.oldrev != '0' * 40):
zuul_params['oldrev'] = item.change.oldrev
if (hasattr(item.change, 'newrev') and item.change.newrev
and item.change.newrev != '0' * 40):
zuul_params['newrev'] = item.change.newrev
zuul_params['projects'] = {} # Set below
zuul_params['items'] = dependent_changes
zuul_params['child_jobs'] = list(item.job_graph.getDirectDependentJobs(
job.name))
params = dict()
params['job'] = job.name
params['timeout'] = job.timeout
params['post_timeout'] = job.post_timeout
params['items'] = merger_items
params['projects'] = []
if hasattr(item.change, 'branch'):
params['branch'] = item.change.branch
else:
params['branch'] = None
params['override_branch'] = job.override_branch
params['override_checkout'] = job.override_checkout
params['repo_state'] = item.current_build_set.repo_state
params['ansible_version'] = job.ansible_version
def make_playbook(playbook):
d = playbook.toDict()
for role in d['roles']:
if role['type'] != 'zuul':
continue
project_metadata = item.layout.getProjectMetadata(
role['project_canonical_name'])
if project_metadata:
role['project_default_branch'] = \
project_metadata.default_branch
else:
role['project_default_branch'] = 'master'
role_trusted, role_project = item.layout.tenant.getProject(
role['project_canonical_name'])
role_connection = role_project.source.connection
role['connection'] = role_connection.connection_name
role['project'] = role_project.name
return d
if job.name != 'noop':
params['playbooks'] = [make_playbook(x) for x in job.run]
params['pre_playbooks'] = [make_playbook(x) for x in job.pre_run]
params['post_playbooks'] = [make_playbook(x) for x in job.post_run]
params['cleanup_playbooks'] = [make_playbook(x)
for x in job.cleanup_run]
nodes = []
for node in nodeset.getNodes():
n = node.toDict()
n.update(dict(name=node.name, label=node.label))
nodes.append(n)
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['ssh_keys'] = []
if pipeline.post_review:
params['ssh_keys'].append(dict(
name='%s project key' % item.change.project.canonical_name,
key=item.change.project.private_ssh_key))
params['vars'] = job.variables
params['extra_vars'] = job.extra_variables
params['host_vars'] = job.host_variables
params['group_vars'] = job.group_variables
params['zuul'] = zuul_params
projects = set()
required_projects = set()
def make_project_dict(project, override_branch=None,
override_checkout=None):
project_metadata = item.layout.getProjectMetadata(
project.canonical_name)
if project_metadata:
project_default_branch = project_metadata.default_branch
else:
project_default_branch = 'master'
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
canonical_name=project.canonical_name,
override_branch=override_branch,
override_checkout=override_checkout,
default_branch=project_default_branch)
if job.required_projects:
for job_project in job.required_projects.values():
(trusted, project) = tenant.getProject(
job_project.project_name)
if project is None:
raise Exception("Unknown project %s" %
(job_project.project_name,))
params['projects'].append(
make_project_dict(project,
job_project.override_branch,
job_project.override_checkout))
projects.add(project)
required_projects.add(project)
for change in dependent_changes:
# We have to find the project this way because it may not
# be registered in the tenant (ie, a foreign project).
source = self.sched.connections.getSourceByCanonicalHostname(
change['project']['canonical_hostname'])
project = source.getProject(change['project']['name'])
if project not in projects:
params['projects'].append(make_project_dict(project))
projects.add(project)
for p in projects:
zuul_params['projects'][p.canonical_name] = (dict(
name=p.name,
short_name=p.name.split('/')[-1],
# Duplicate this into the dict too, so that iterating
# project.values() is easier for callers
canonical_name=p.canonical_name,
canonical_hostname=p.canonical_hostname,
src_dir=os.path.join('src', p.canonical_name),
required=(p in required_projects),
))
params['zuul_event_id'] = item.event.zuul_event_id
build = Build(job, uuid, zuul_event_id=item.event.zuul_event_id)
build.parameters = params
build.nodeset = nodeset
log.debug("Adding build %s of job %s to item %s",
build, job, item)
item.addBuild(build)
if job.name == 'noop':
self.sched.onBuildStarted(build)
self.sched.onBuildCompleted(build, 'SUCCESS', {}, [])
return build
# Update zuul attempts after addBuild above to ensure build_set
# is up to date.
attempts = build.build_set.getTries(job.name)
zuul_params['attempts'] = attempts
functions = getGearmanFunctions(self.gearman)
function_name = 'executor:execute'
# Because all nodes belong to the same provider, region and
# availability zone we can get executor_zone from only the first
# node.
executor_zone = None
if nodes and nodes[0].get('attributes'):
executor_zone = nodes[0]['attributes'].get('executor-zone')
if executor_zone:
_fname = '%s:%s' % (
function_name,
executor_zone)
if _fname in functions:
function_name = _fname
else:
self.log.warning(
"Job requested '%s' zuul-executor zone, but no "
"zuul-executors found for this zone; ignoring zone "
"request" % executor_zone)
gearman_job = gear.TextJob(
function_name, json_dumps(params), unique=uuid)
build.__gearman_job = gearman_job
build.__gearman_worker = None
self.builds[uuid] = build
if pipeline.precedence == zuul.model.PRECEDENCE_NORMAL:
precedence = gear.PRECEDENCE_NORMAL
elif pipeline.precedence == zuul.model.PRECEDENCE_HIGH:
precedence = gear.PRECEDENCE_HIGH
elif pipeline.precedence == zuul.model.PRECEDENCE_LOW:
precedence = gear.PRECEDENCE_LOW
try:
self.gearman.submitJob(gearman_job, precedence=precedence,
timeout=300)
except Exception:
log.exception("Unable to submit job to Gearman")
self.onBuildCompleted(gearman_job, 'EXCEPTION')
return build
if not gearman_job.handle:
log.error("No job handle was received for %s after"
" 300 seconds; marking as lost.",
gearman_job)
self.onBuildCompleted(gearman_job, 'NO_HANDLE')
log.debug("Received handle %s for %s", gearman_job.handle, build)
return build
def execute(self, job, item, pipeline, dependent_changes=[],
merger_items=[]):
tenant = pipeline.tenant
uuid = str(uuid4().hex)
nodeset = item.current_build_set.getJobNodeSet(job.name)
self.log.info(
"Execute job %s (uuid: %s) on nodes %s for change %s "
"with dependent changes %s" % (
job, uuid, nodeset, item.change, dependent_changes))
project = dict(
name=item.change.project.name,
short_name=item.change.project.name.split('/')[-1],
canonical_hostname=item.change.project.canonical_hostname,
canonical_name=item.change.project.canonical_name,
src_dir=os.path.join('src', item.change.project.canonical_name),
)
zuul_params = dict(build=uuid,
buildset=item.current_build_set.uuid,
ref=item.change.ref,
pipeline=pipeline.name,
job=job.name,
voting=job.voting,
project=project,
tenant=tenant.name,
timeout=job.timeout,
jobtags=sorted(job.tags),
_inheritance_path=list(job.inheritance_path))
if job.artifact_data:
zuul_params['artifacts'] = job.artifact_data
if job.override_checkout:
zuul_params['override_checkout'] = job.override_checkout
if hasattr(item.change, 'branch'):
zuul_params['branch'] = item.change.branch
if hasattr(item.change, 'tag'):
zuul_params['tag'] = item.change.tag
if hasattr(item.change, 'number'):
zuul_params['change'] = str(item.change.number)
if hasattr(item.change, 'url'):
zuul_params['change_url'] = item.change.url
if hasattr(item.change, 'patchset'):
zuul_params['patchset'] = str(item.change.patchset)
if hasattr(item.change, 'message'):
zuul_params['message'] = item.change.message
if (hasattr(item.change, 'oldrev') and item.change.oldrev
and item.change.oldrev != '0' * 40):
zuul_params['oldrev'] = item.change.oldrev
if (hasattr(item.change, 'newrev') and item.change.newrev
and item.change.newrev != '0' * 40):
zuul_params['newrev'] = item.change.newrev
zuul_params['projects'] = {} # Set below
zuul_params['items'] = dependent_changes
zuul_params['child_jobs'] = list(item.job_graph.getDirectDependentJobs(
job.name))
params = dict()
params['job'] = job.name
params['timeout'] = job.timeout
params['post_timeout'] = job.post_timeout
params['items'] = merger_items
params['projects'] = []
if hasattr(item.change, 'branch'):
params['branch'] = item.change.branch
else:
params['branch'] = None
params['override_branch'] = job.override_branch
params['override_checkout'] = job.override_checkout
params['repo_state'] = item.current_build_set.repo_state
params['ansible_version'] = job.ansible_version
def make_playbook(playbook):
d = playbook.toDict()
for role in d['roles']:
if role['type'] != 'zuul':
continue
project_metadata = item.layout.getProjectMetadata(
role['project_canonical_name'])
if project_metadata:
role['project_default_branch'] = \
project_metadata.default_branch
else:
role['project_default_branch'] = 'master'
role_trusted, role_project = item.layout.tenant.getProject(
role['project_canonical_name'])
role_connection = role_project.source.connection
role['connection'] = role_connection.connection_name
role['project'] = role_project.name
return d
if job.name != 'noop':
params['playbooks'] = [make_playbook(x) for x in job.run]
params['pre_playbooks'] = [make_playbook(x) for x in job.pre_run]
params['post_playbooks'] = [make_playbook(x) for x in job.post_run]
nodes = []
for node in nodeset.getNodes():
n = node.toDict()
n.update(dict(name=node.name, label=node.label))
nodes.append(n)
params['nodes'] = nodes
params['groups'] = [group.toDict() for group in nodeset.getGroups()]
params['ssh_keys'] = []
if pipeline.post_review:
params['ssh_keys'].append(dict(
name='%s project key' % item.change.project.canonical_name,
key=item.change.project.private_ssh_key))
params['vars'] = job.variables
params['extra_vars'] = job.extra_variables
params['host_vars'] = job.host_variables
params['group_vars'] = job.group_variables
params['zuul'] = zuul_params
projects = set()
required_projects = set()
def make_project_dict(project, override_branch=None,
override_checkout=None):
project_metadata = item.layout.getProjectMetadata(
project.canonical_name)
if project_metadata:
project_default_branch = project_metadata.default_branch
else:
project_default_branch = 'master'
connection = project.source.connection
return dict(connection=connection.connection_name,
name=project.name,
canonical_name=project.canonical_name,
override_branch=override_branch,
override_checkout=override_checkout,
default_branch=project_default_branch)
if job.required_projects:
for job_project in job.required_projects.values():
(trusted, project) = tenant.getProject(
job_project.project_name)
if project is None:
raise Exception("Unknown project %s" %
(job_project.project_name,))
params['projects'].append(
make_project_dict(project,
job_project.override_branch,
job_project.override_checkout))
projects.add(project)
required_projects.add(project)
for change in dependent_changes:
# We have to find the project this way because it may not
# be registered in the tenant (ie, a foreign project).
source = self.sched.connections.getSourceByCanonicalHostname(
change['project']['canonical_hostname'])
project = source.getProject(change['project']['name'])
if project not in projects:
params['projects'].append(make_project_dict(project))
projects.add(project)
for p in projects:
zuul_params['projects'][p.canonical_name] = (dict(
name=p.name,
short_name=p.name.split('/')[-1],
# Duplicate this into the dict too, so that iterating
# project.values() is easier for callers
canonical_name=p.canonical_name,
canonical_hostname=p.canonical_hostname,
src_dir=os.path.join('src', p.canonical_name),
required=(p in required_projects),
))
build = Build(job, uuid)
build.parameters = params
build.nodeset = nodeset
self.log.debug("Adding build %s of job %s to item %s" %
(build, job, item))
item.addBuild(build)
if job.name == 'noop':
self.sched.onBuildStarted(build)
self.sched.onBuildCompleted(build, 'SUCCESS', {}, [])
return build
functions = getGearmanFunctions(self.gearman)
function_name = 'executor:execute'
# Because all nodes belong to the same provider, region and
# availability zone we can get executor_zone from only the first
# node.
executor_zone = None
if nodes and nodes[0].get('attributes'):
executor_zone = nodes[0]['attributes'].get('executor-zone')
if executor_zone:
_fname = '%s:%s' % (
function_name,
executor_zone)
if _fname in functions:
function_name = _fname
else:
self.log.warning(
"Job requested '%s' zuul-executor zone, but no "
"zuul-executors found for this zone; ignoring zone "
"request" % executor_zone)
gearman_job = gear.TextJob(
function_name, json_dumps(params), unique=uuid)
build.__gearman_job = gearman_job
build.__gearman_worker = None
self.builds[uuid] = build
if pipeline.precedence == zuul.model.PRECEDENCE_NORMAL:
precedence = gear.PRECEDENCE_NORMAL
elif pipeline.precedence == zuul.model.PRECEDENCE_HIGH:
precedence = gear.PRECEDENCE_HIGH
elif pipeline.precedence == zuul.model.PRECEDENCE_LOW:
precedence = gear.PRECEDENCE_LOW
try:
self.gearman.submitJob(gearman_job, precedence=precedence,
timeout=300)
except Exception:
self.log.exception("Unable to submit job to Gearman")
self.onBuildCompleted(gearman_job, 'EXCEPTION')
return build
if not gearman_job.handle:
self.log.error("No job handle was received for %s after"
" 300 seconds; marking as lost." %
gearman_job)
self.onBuildCompleted(gearman_job, 'NO_HANDLE')
self.log.debug("Received handle %s for %s" % (gearman_job.handle,
build))
return build