def trigger_if_necessary(da, pkg, rosdistro,
jenkins_instance, missing_by_arch):
if da != 'source' and 'source' in missing_by_arch and \
pkg in missing_by_arch['source']:
print (" Skipping trigger of binarydeb job for package '%s' on arch '%s' as the sourcedeb job will trigger them automatically" % (pkg, da))
return False
if da == 'source':
job_name = '%s_sourcedeb' % (debianize_package_name(rosdistro, pkg))
else:
job_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro, pkg),
da)
job_info = jenkins_instance.get_job_info(job_name)
if 'color' in job_info and 'anime' in job_info['color']:
print (" Skipping trigger of job %s because it's already running" % job_name)
return False
if 'inQueue' in job_info and job_info['inQueue']:
print (" Skipping trigger of job '%s' because it's already queued" % job_name)
return False
if da != 'source' and 'upstreamProjects' in job_info:
upstream = job_info['upstreamProjects']
for p in missing_by_arch[da]:
p_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro, p),
da)
for u in upstream:
if u['name'] == p_name:
print (" Skipping trigger of job '%s' because the upstream job '%s' is also triggered" % (job_name, p_name))
return False
print ("Triggering '%s'" % (job_name))
#return jenkins_instance.build_job(job_name)
# replicate internal implementation of Jenkins.build_job()
import urllib2
if not jenkins_instance.job_exists(job_name):
raise jenkins.JenkinsException('no such job[%s]' % (job_name))
# pass parameters to create a POST request instead of GET
return jenkins_instance.jenkins_open(urllib2.Request(jenkins_instance.build_job_url(job_name), 'foo=bar'))
pp = pprint.PrettyPrinter()
print("net Missing")
pp.pprint(missing)
if args.commit:
jenkins_instance = jenkins_support.JenkinsConfig_to_handle(
jenkins_support.load_server_config_file(jenkins_support.get_default_catkin_debs_config())
)
for s, dist_archs in missing.iteritems():
detected_source = False
for da in dist_archs:
if "source" in da:
print("Source missing for %s" % da)
detected_source = True
if detected_source:
job_name = "%s_sourcedeb" % (debianize_package_name(args.rosdistro, s))
trigger_if_not_building(job_name, jenkins_instance)
print("Skipping debbuilds for this package [%s] as the sourcedeb will trigger them automatically" % s)
continue
for da in dist_archs:
job_name = "%s_binarydeb_%s" % (debianize_package_name(args.rosdistro, s), da)
trigger_if_not_building(job_name, jenkins_instance)
else:
print('This was not pushed to the server. If you want to do so use "--commit" to do it for real.')
def debianize_missing(missing):
debianized_missing = {}
for s, v in missing.iteritems():
debianized_missing[debianize_package_name(args.rosdistro, s)] = v
return debianized_missing
workspace = args.repos
try:
if not args.repos:
workspace = tempfile.mkdtemp()
(dependencies, package_names_by_url) = dependency_walker.get_dependencies(workspace, repo_map['repositories'], args.rosdistro)
dry_jobgraph = release_jobs.dry_generate_jobgraph(args.rosdistro)
combined_jobgraph = {}
for k, v in dependencies.iteritems():
combined_jobgraph[k] = v
for k, v in dry_jobgraph.iteritems():
combined_jobgraph[k] = v
# setup a job triggered by all other debjobs
combined_jobgraph[debianize_package_name(args.rosdistro, 'metapackages')] = combined_jobgraph.keys()
finally:
if not args.repos:
shutil.rmtree(workspace)
missing = release_jobs.compute_missing(
[],
args.fqdn,
rosdistro=args.rosdistro)
#print (display_missing_table(missing))
debianized_missing = debianize_missing(missing)
if args.commit:
jenkins_instance = jenkins_support.JenkinsConfig_to_handle(
jenkins_support.load_server_config_file(
jenkins_support.get_default_catkin_debs_config()))
for s, dist_archs in missing.iteritems():
detected_source = False
for da in dist_archs:
if 'source' in da:
print("Source missing for %s" % da)
detected_source = True
if detected_source:
job_name = '%s_sourcedeb' % (debianize_package_name(
args.rosdistro, s))
trigger_if_not_building(job_name, jenkins_instance)
print(
"Skipping debbuilds for this package [%s] as the sourcedeb will trigger them automatically"
% s)
continue
for da in dist_archs:
job_name = '%s_binarydeb_%s' % (debianize_package_name(
args.rosdistro, s), da)
trigger_if_not_building(job_name, jenkins_instance)
else:
print(
'This was not pushed to the server. If you want to do so use "--commit" to do it for real.'
def doit(repo_map, package_names_by_url, distros, fqdn, jobs_graph, rosdistro, commit=False, delete_extra_jobs=False):
jenkins_instance = None
if args.commit or delete_extra_jobs:
jenkins_instance = jenkins_support.JenkinsConfig_to_handle(jenkins_support.load_server_config_file(jenkins_support.get_default_catkin_debs_config()))
# What ROS distro are we configuring?
rosdistro = repo_map['release-name']
# Figure out default distros. Command-line arg takes precedence; if
# it's not specified, then read targets.yaml.
if distros:
default_distros = distros
else:
print('Fetching "%s"' % (URL_PROTOTYPE % 'targets'))
targets_map = yaml.load(urllib2.urlopen(URL_PROTOTYPE % 'targets'))
my_targets = [x for x in targets_map if rosdistro in x]
if len(my_targets) != 1:
print('Must have exactly one entry for rosdistro "%s" in targets.yaml' % rosdistro)
sys.exit(1)
default_distros = my_targets[0][rosdistro]
# We take the intersection of repo-specific targets with default
# targets.
results = {}
for short_package_name, r in sorted(repo_map['repositories'].items()):
if 'url' not in r:
print('"url" key missing for repository "%s"; skipping' % r)
continue
url = r['url']
if url not in package_names_by_url:
print('Repo "%s" is missing from the list; must have been skipped (e.g., for missing a stack.xml)' % r)
continue
if 'target' not in r or r['target'] == 'all':
target_distros = default_distros
else:
target_distros = list(set(r['target']) & set(default_distros))
print ('Configuring WET stack "%s" for "%s"' % (r['url'], target_distros))
results[package_names_by_url[url]] = release_jobs.doit(url,
package_names_by_url[url],
target_distros,
fqdn,
jobs_graph,
rosdistro=rosdistro,
short_package_name=short_package_name,
commit=commit,
jenkins_instance=jenkins_instance)
time.sleep(1)
#print ('individual results', results[package_names_by_url[url]])
if args.wet_only:
print ("wet only selected, skipping dry and delete")
return results
if rosdistro == 'backports':
print ("no dry backports stopping execution")
return results
#dry stacks
# dry dependencies
d = rospkg.distro.load_distro(rospkg.distro.distro_uri(rosdistro))
for s in d.stacks:
print ("Configuring DRY job [%s]" % s)
results[debianize_package_name(rosdistro, s) ] = release_jobs.dry_doit(s, default_distros, rosdistro, jobgraph=jobs_graph, commit=commit, jenkins_instance=jenkins_instance)
time.sleep(1)
# special metapackages job
results[debianize_package_name(rosdistro, 'metapackages') ] = release_jobs.dry_doit('metapackages', default_distros, rosdistro, jobgraph=jobs_graph, commit=commit, jenkins_instance=jenkins_instance)
if delete_extra_jobs:
# clean up extra jobs
configured_jobs = set()
for _, v in results.iteritems():
release_jobs.summarize_results(*v)
for e in v:
configured_jobs.update(set(e))
existing_jobs = set([j['name'] for j in jenkins_instance.get_jobs()])
relevant_jobs = existing_jobs - configured_jobs
relevant_jobs = [j for j in relevant_jobs if rosdistro in j and ('sourcedeb' in j or 'binarydeb' in j)]
for j in relevant_jobs:
print('Job "%s" detected as extra' % j)
if commit:
jenkins_instance.delete_job(j)
print('Deleted job "%s"' % j)
return results
workspace = args.repos
try:
if not args.repos:
workspace = tempfile.mkdtemp()
(dependencies, package_names_by_url) = dependency_walker.get_dependencies(workspace, repo_map['repositories'], args.rosdistro)
dry_jobgraph = release_jobs.dry_generate_jobgraph(args.rosdistro)
combined_jobgraph = {}
for k, v in dependencies.iteritems():
combined_jobgraph[k] = v
for k, v in dry_jobgraph.iteritems():
combined_jobgraph[k] = v
# setup a job triggered by all other debjobs
combined_jobgraph[debianize_package_name(args.rosdistro, 'metapackages')] = combined_jobgraph.keys()
finally:
if not args.repos:
shutil.rmtree(workspace)
results_map = doit(repo_map,
package_names_by_url,
args.distros,
args.fqdn,
combined_jobgraph,
rosdistro=args.rosdistro,
commit=args.commit,
delete_extra_jobs=args.delete)
def debianize_missing(missing):
debianized_missing = {}
for s, v in missing.iteritems():
debianized_missing[debianize_package_name(args.rosdistro, s)] = v
return debianized_missing