def trigger_if_necessary(da, pkg, rosdistro, jenkins_instance, missing_by_arch): if da != 'source' and 'source' in missing_by_arch and pkg in missing_by_arch['source']: print (" Skipping trigger of binarydeb job for package '%s' on arch '%s' as the sourcedeb job will trigger them automatically" % (pkg, da)) return False if da == 'source': job_name = '%s_sourcedeb' % (debianize_package_name(rosdistro, pkg)) else: job_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro, pkg), da) job_info = jenkins_instance.get_job_info(job_name) if 'color' in job_info and 'anime' in job_info['color']: print (" Skipping trigger of job %s because it's already running" % job_name) return False if 'inQueue' in job_info and job_info['inQueue']: print (" Skipping trigger of job '%s' because it's already queued" % job_name) return False if da != 'source' and 'upstreamProjects' in job_info: upstream = job_info['upstreamProjects'] for p in missing_by_arch[da]: p_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro, p), da) for u in upstream: if u['name'] == p_name: print (" Skipping trigger of job '%s' because the upstream job '%s' is also triggered" % (job_name, p_name)) return False print ("Triggering '%s'" % (job_name)) jenkins_instance.build_job(job_name) return True
def get_dependencies(rd, packages): dependencies = {} v = rospack.init_rospack_interface() for p in packages: deps = rd.get_depends(p) dp = debianize_package_name(rd.name, p) dependencies[dp] = [] combined_deps = set(deps['build']) | set(deps['run']) for d in combined_deps: if not rospack.is_system_dependency(v, d): dependencies[dp].append(debianize_package_name(rd.name, d)) return dependencies
def trigger_if_necessary(da, pkg, rosdistro, jenkins_instance, missing_by_arch): if da != 'source' and 'source' in missing_by_arch and pkg in missing_by_arch['source']: print (" Skipping trigger of binarydeb job for package '%s' on arch '%s' as the sourcedeb job will trigger them automatically" % (pkg, da)) return False if da == 'source': job_name = '%s_sourcedeb' % (debianize_package_name(rosdistro, pkg)) else: job_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro, pkg), da) job_info = jenkins_instance.get_job_info(job_name) if 'color' in job_info and 'anime' in job_info['color']: print (" Skipping trigger of job %s because it's already running" % job_name) return False if 'inQueue' in job_info and job_info['inQueue']: print (" Skipping trigger of job '%s' because it's already queued" % job_name) return False if da != 'source' and 'upstreamProjects' in job_info: upstream = job_info['upstreamProjects'] for p in missing_by_arch[da]: p_name = '%s_binarydeb_%s' % (debianize_package_name(rosdistro, p), da) for u in upstream: if u['name'] == p_name: print (" Skipping trigger of job '%s' because the upstream job '%s' is also triggered" % (job_name, p_name)) return False print ("Triggering '%s'" % (job_name)) #return jenkins_instance.build_job(job_name) # replicate internal implementation of Jenkins.build_job() import urllib2 if not jenkins_instance.job_exists(job_name): raise jenkins.JenkinsException('no such job[%s]' % (job_name)) # pass parameters to create a POST request instead of GET return jenkins_instance.jenkins_open(urllib2.Request(jenkins_instance.build_job_url(job_name), 'foo=bar'))
def doit(job_params, dry_maintainers, packages, rosdist_rep, wet_only=False, commit = False, delete_extra_jobs = False, whitelist_repos = None): jenkins_instance = None if commit or delete_extra_jobs: jenkins_config = jenkins_support.load_server_config_file(jenkins_support.get_default_catkin_debs_config()) jenkins_instance = jenkins_support.JenkinsConfig_to_handle(jenkins_config) rosdistro = job_params.rosdistro rd = job_params.rd # We take the intersection of repo-specific targets with default # targets. results = {} for repo_name in sorted(rd.get_repositories()): if whitelist_repos and repo_name not in whitelist_repos: continue r = rd.get_repository(repo_name) print ('Configuring WET repo "%s" at "%s" for "%s"' % (r.name, r.url, job_params.distros)) p_list = [p.name for p in r.packages] for p in sorted(p_list): if not r.version: print('- skipping "%s" since version is null' % p) continue pkg_name = debianize_package_name(rosdistro, p) maintainers = rd.get_maintainers(p) pp = PackageParams(package_name=pkg_name, package=packages[p], release_uri=r.url, short_package_name=p, maintainers=maintainers) results[pkg_name] = release_jobs.doit(job_params=job_params, pkg_params=pp, commit=commit, jenkins_instance=jenkins_instance) #time.sleep(1) #print ('individual results', results[pkg_name]) if wet_only: print ("wet only selected, skipping dry and delete") return results default_distros = job_params.distros target_arches = list(set([x for d in default_distros for x in job_params.arches[d]])) rosdistro = job_params.rosdistro jobs_graph = job_params.jobgraph if rosdistro == 'backports': print ("No dry backports support") return results if rosdistro == 'fuerte': packages_for_sync = 300 elif rosdistro == 'groovy': packages_for_sync = 500 elif rosdistro == 'hydro': packages_for_sync = 60 else: packages_for_sync = 10000 #dry stacks # dry dependencies d = rospkg.distro.load_distro(rospkg.distro.distro_uri(rosdistro)) for s in sorted(d.stacks.iterkeys()): if whitelist_repos and s not in whitelist_repos: continue print ("Configuring DRY job [%s]" % s) if not d.stacks[s].version: print('- skipping "%s" since version is null' % s) continue results[debianize_package_name(rd.name, s)] = release_jobs.dry_doit(s, dry_maintainers[s], default_distros, target_arches, rosdistro, jobgraph=jobs_graph, commit=commit, jenkins_instance=jenkins_instance, packages_for_sync=packages_for_sync) #time.sleep(1) # special metapackages job if not whitelist_repos or 'metapackages' in whitelist_repos: results[debianize_package_name(rd.name, 'metapackages')] = release_jobs.dry_doit('metapackages', [], default_distros, target_arches, rosdistro, jobgraph=jobs_graph, commit=commit, jenkins_instance=jenkins_instance, packages_for_sync=packages_for_sync) if delete_extra_jobs: assert(not whitelist_repos) # clean up extra jobs configured_jobs = set() for jobs in results.values(): release_jobs.summarize_results(*jobs) for e in jobs: configured_jobs.update(set(e)) existing_jobs = set([j['name'] for j in jenkins_instance.get_jobs()]) relevant_jobs = existing_jobs - configured_jobs relevant_jobs = [j for j in relevant_jobs if rosdistro in j and ('_sourcedeb' in j or '_binarydeb' in j)] for j in relevant_jobs: print('Job "%s" detected as extra' % j) if commit: jenkins_instance.delete_job(j) print('Deleted job "%s"' % j) return results
# even for wet_only the dry packages need to be consider, else they are not added as downstream dependencies for the wet jobs if not args.wet_only: stack_depends, dry_maintainers = release_jobs.dry_get_stack_dependencies(args.rosdistro) dry_jobgraph = release_jobs.dry_generate_jobgraph(args.rosdistro, dependencies, stack_depends) else: dry_maintainers = [] combined_jobgraph = {} for k, v in dependencies.iteritems(): combined_jobgraph[k] = v if not args.wet_only: for k, v in dry_jobgraph.iteritems(): combined_jobgraph[k] = v # setup a job triggered by all other debjobs combined_jobgraph[debianize_package_name(args.rosdistro, 'metapackages')] = combined_jobgraph.keys() targets = get_targets(rd, args.distros, args.arches) jp = JobParams(rosdistro=args.rosdistro, distros=targets.keys(), arches=targets, fqdn=args.fqdn, jobgraph=combined_jobgraph, rosdist_rep=args.rosdist_rep, rd_object=rd) results_map = doit(job_params=jp, packages=packages, dry_maintainers=dry_maintainers, commit=args.commit, wet_only=args.wet_only,
def doit(job_params, dry_maintainers, packages, rosdist_rep, wet_only=False, commit=False, delete_extra_jobs=False, whitelist_repos=None): jenkins_instance = None if commit or delete_extra_jobs: jenkins_config = jenkins_support.load_server_config_file( jenkins_support.get_default_catkin_debs_config()) jenkins_instance = jenkins_support.JenkinsConfig_to_handle( jenkins_config) rosdistro = job_params.rosdistro rd = job_params.rd # We take the intersection of repo-specific targets with default # targets. results = {} for repo_name in sorted(rd.get_repositories()): if whitelist_repos and repo_name not in whitelist_repos: continue r = rd.get_repository(repo_name) print('Configuring WET repo "%s" at "%s" for "%s"' % (r.name, r.url, job_params.distros)) p_list = [p.name for p in r.packages] for p in sorted(p_list): if not r.version: print('- skipping "%s" since version is null' % p) continue pkg_name = debianize_package_name(rosdistro, p) maintainers = rd.get_maintainers(p) pp = PackageParams(package_name=pkg_name, package=packages[p], release_uri=r.url, short_package_name=p, maintainers=maintainers) results[pkg_name] = release_jobs.doit( job_params=job_params, pkg_params=pp, commit=commit, jenkins_instance=jenkins_instance) #time.sleep(1) #print ('individual results', results[pkg_name]) if wet_only: print("wet only selected, skipping dry and delete") return results default_distros = job_params.distros target_arches = list( set([x for d in default_distros for x in job_params.arches[d]])) rosdistro = job_params.rosdistro jobs_graph = job_params.jobgraph if rosdistro == 'backports': print("No dry backports support") return results if rosdistro == 'fuerte': packages_for_sync = 300 elif rosdistro == 'groovy': packages_for_sync = 500 elif rosdistro == 'hydro': packages_for_sync = 60 else: packages_for_sync = 10000 #dry stacks # dry dependencies d = rospkg.distro.load_distro(rospkg.distro.distro_uri(rosdistro)) for s in sorted(d.stacks.iterkeys()): if whitelist_repos and s not in whitelist_repos: continue print("Configuring DRY job [%s]" % s) if not d.stacks[s].version: print('- skipping "%s" since version is null' % s) continue results[debianize_package_name(rd.name, s)] = release_jobs.dry_doit( s, dry_maintainers[s], default_distros, target_arches, rosdistro, jobgraph=jobs_graph, commit=commit, jenkins_instance=jenkins_instance, packages_for_sync=packages_for_sync) #time.sleep(1) # special metapackages job if not whitelist_repos or 'metapackages' in whitelist_repos: results[debianize_package_name( rd.name, 'metapackages')] = release_jobs.dry_doit( 'metapackages', [], default_distros, target_arches, rosdistro, jobgraph=jobs_graph, commit=commit, jenkins_instance=jenkins_instance, packages_for_sync=packages_for_sync) if delete_extra_jobs: assert (not whitelist_repos) # clean up extra jobs configured_jobs = set() for jobs in results.values(): release_jobs.summarize_results(*jobs) for e in jobs: configured_jobs.update(set(e)) existing_jobs = set([j['name'] for j in jenkins_instance.get_jobs()]) relevant_jobs = existing_jobs - configured_jobs relevant_jobs = [ j for j in relevant_jobs if rosdistro in j and ('_sourcedeb' in j or '_binarydeb' in j) ] for j in relevant_jobs: print('Job "%s" detected as extra' % j) if commit: jenkins_instance.delete_job(j) print('Deleted job "%s"' % j) return results
stack_depends, dry_maintainers = release_jobs.dry_get_stack_dependencies( args.rosdistro) dry_jobgraph = release_jobs.dry_generate_jobgraph( args.rosdistro, dependencies, stack_depends) else: dry_maintainers = [] combined_jobgraph = {} for k, v in dependencies.iteritems(): combined_jobgraph[k] = v if not args.wet_only: for k, v in dry_jobgraph.iteritems(): combined_jobgraph[k] = v # setup a job triggered by all other debjobs combined_jobgraph[debianize_package_name( args.rosdistro, 'metapackages')] = combined_jobgraph.keys() targets = get_targets(rd, args.distros, args.arches) jp = JobParams(rosdistro=args.rosdistro, distros=targets.keys(), arches=targets, fqdn=args.fqdn, jobgraph=combined_jobgraph, rosdist_rep=args.rosdist_rep, rd_object=rd) results_map = doit(job_params=jp, packages=packages, dry_maintainers=dry_maintainers, commit=args.commit, wet_only=args.wet_only,