def __init__(self, rosdistro_name):
self.packages = {}
from buildfarm.ros_distro import Rosdistro
# for fuerte we still fetch the new groovy rosdistro to get a list of distros
rd = Rosdistro(rosdistro_name if rosdistro_name != 'fuerte' else 'groovy')
self.rosdistro_index = rd._index
self.rosdistro_dist = rd._dist
# load wet rosdistro packages
if rosdistro_name == 'fuerte':
from buildfarm.ros_distro_fuerte import Rosdistro as RosdistroFuerte
rd = RosdistroFuerte(rosdistro_name)
for pkg_name in rd.get_package_list():
version = rd.get_version(pkg_name, full_version=True)
if version:
self.packages[pkg_name] = RosdistroVersion(pkg_name, 'wet', version)
# load dry rosdistro stacks
if rosdistro_name == 'groovy':
dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro_name)))
stacks = dry_yaml['stacks'] or {}
for stack_name, d in stacks.items():
if stack_name == '_rules':
continue
version = d.get('version')
if version:
if stack_name in self.packages:
logging.warn("Stack '%s' exists in dry (%s) as well as in wet (%s) distro. Ignoring dry package." % (stack_name, version, self.packages[stack_name].version))
continue
self.packages[stack_name] = RosdistroVersion(stack_name, 'dry', version)
# load variants
variants = dry_yaml['variants'] or {}
for variant in variants:
if len(variant) != 1:
logging.warn("Not length 1 dict in variant '%s': skipping" % variant)
continue
variant_name = variant.keys()[0]
if variant_name in self.packages:
logging.warn("Variant '%s' exists also as a package in %s. Ignoring variant." % (variant_name, self.packages[variant_name].type))
continue
self.packages[variant_name] = RosdistroVersion(variant_name, 'variant', '1.0.0')
def __init__(self, rosdistro_name):
self.packages = {}
from buildfarm.ros_distro import Rosdistro
# for fuerte we still fetch the new groovy rosdistro to get a list of distros
rd = Rosdistro(rosdistro_name if rosdistro_name != 'fuerte' else 'groovy')
self.rosdistro_index = rd._index
self.rosdistro_dist = rd._dist
# load wet rosdistro packages
if rosdistro_name == 'fuerte':
from buildfarm.ros_distro_fuerte import Rosdistro as RosdistroFuerte
rd = RosdistroFuerte(rosdistro_name)
for pkg_name in rd.get_package_list():
version = rd.get_version(pkg_name, full_version=True)
if version:
self.packages[pkg_name] = RosdistroVersion(pkg_name, 'wet', version)
# load dry rosdistro stacks
if rosdistro_name == 'groovy':
dry_yaml = yaml.load(urllib2.urlopen(distro_uri(rosdistro_name)))
stacks = dry_yaml['stacks'] or {}
for stack_name, d in stacks.items():
if stack_name == '_rules':
continue
version = d.get('version')
if version:
if stack_name in self.packages:
logging.warn("Stack '%s' exists in dry (%s) as well as in wet (%s) distro. Ignoring dry package." % (stack_name, version, self.packages[stack_name].version))
continue
self.packages[stack_name] = RosdistroVersion(stack_name, 'dry', version)
# load variants
variants = dry_yaml['variants'] or {}
for variant in variants:
if len(variant) != 1:
logging.warn("Not length 1 dict in variant '%s': skipping" % variant)
continue
variant_name = variant.keys()[0]
if variant_name in self.packages:
logging.warn("Variant '%s' exists also as a package in %s. Ignoring variant." % (variant_name, self.packages[variant_name].type))
continue
self.packages[variant_name] = RosdistroVersion(variant_name, 'variant', '1.0.0')
(returncode, result, message) = run_shell_command('debuild binary', repo_path, shell=True, show_stdout=False)
if returncode != 0:
raise RosGitBuildError("%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild binary', message))
deb_files = glob.glob(os.path.join(repo_path, '..', '%s*.deb' % (deb_package_name + '_' + rd_obj.get_version(package_name, full_version=True))))
if len(deb_files) > 0:
# install the deb
from apt.debfile import DebPackage
deb_pkg = DebPackage(deb_files[0])
deb_pkg.check()
packages_needed = ' '.join(deb_pkg.missing_deps)
(returncode, result, message) = run_shell_command('sudo apt-get -y install %s' % packages_needed, shell=True, show_stdout=True)
if returncode != 0:
raise RosGitBuildError("%s!!! Error building %s: can't install dependent packages %s" % (level_prefix, deb_package_name, packages_needed))
(returncode, result, message) = run_shell_command('sudo dpkg -i %s' % deb_files[0], shell=True, show_stdout=True)
if returncode != 0:
raise RosGitBuildError("%s!!! Error building %s from %s: %s \n %s" % (level_prefix, deb_package_name, package_fetcher.url(package_name), 'debuild binary', message))
else:
raise RosGitBuildError("%s!!! Can't find a built debian package for %s after the build!" % (level_prefix, deb_package_name))
if __name__ == '__main__':
args = parse_options()
rd = Rosdistro(args.distro)
fetch = VcsPackageFetcher(rd, args.workspace)
cache = apt.Cache()
for package in args.packages:
if package in rd.get_package_list():
build_debian_package(fetch, package, cache, rd, get_dependencies=True)
else:
print('!!! Skipping nonexistent package %s' % (package))
print('Deleted job "%s"' % j)
return results
if __name__ == '__main__':
args = parse_options()
print('Loading rosdistro %s' % args.rosdistro)
workspace = args.repo_workspace
if not workspace:
workspace = os.path.join(tempfile.gettempdir(), 'repo-workspace-%s' % args.rosdistro)
from buildfarm.ros_distro import Rosdistro
rd = Rosdistro(args.rosdistro)
from buildfarm import dependency_walker
packages = dependency_walker.get_packages(workspace, rd, skip_update=args.skip_update)
dependencies = dependency_walker.get_jenkins_dependencies(args.rosdistro, packages)
yum_target_repository = rd._build_files[0].yum_target_repository
if args.fqdn is None:
fqdn_parts = urlsplit(yum_target_repository)
args.fqdn = fqdn_parts.netloc
if args.arches is None:
args.arches = rd.get_arches()
release_jobs.check_for_circular_dependencies(dependencies)
# even for wet_only the dry packages need to be consider, else they are not added as downstream dependencies for the wet jobs
stack_depends, dry_maintainers = release_jobs.dry_get_stack_dependencies(args.rosdistro)
% (level_prefix, deb_package_name, packages_needed))
(returncode, result,
message) = run_shell_command('sudo dpkg -i %s' % deb_files[0],
shell=True,
show_stdout=True)
if returncode != 0:
raise RosGitBuildError(
"%s!!! Error building %s from %s: %s \n %s" %
(level_prefix, deb_package_name,
package_fetcher.url(package_name), 'debuild binary', message))
else:
raise RosGitBuildError(
"%s!!! Can't find a built debian package for %s after the build!" %
(level_prefix, deb_package_name))
if __name__ == '__main__':
args = parse_options()
rd = Rosdistro(args.distro)
fetch = VcsPackageFetcher(rd, args.workspace)
cache = apt.Cache()
for package in args.packages:
if package in rd.get_package_list():
build_debian_package(fetch,
package,
cache,
rd,
get_dependencies=True)
else:
print('!!! Skipping nonexistent package %s' % (package))
def gen_metapkgs(distro,
os_platform,
arch,
staging_dir,
repo_fqdn,
force=False):
distro_name = distro.release_name
# Retrieve the package list from the shadow repo
packageurl = repo_url(
repo_fqdn
) + "/dists/%(os_platform)s/main/binary-%(arch)s/Packages" % locals()
try:
packagetxt = urllib2.urlopen(packageurl).read()
except urllib2.URLError as ex:
print("Failed to open url: %s with error %s" % (packageurl, ex))
raise ex
except:
print("Failed to open url: %s" % (packageurl))
raise
packagelist = parse_deb_packages(packagetxt)
debs = []
missing = []
_, _, missing_excluded, missing_excluded_dep = list_missing.get_missing(
distro, os_platform, arch, repo=repo_url(repo_fqdn))
missing_ok = missing_excluded.union(missing_excluded_dep)
# if (metapkg missing) or (metapkg missing deps), then create
# modify create to version-lock deps
if distro_name != 'fuerte':
from buildfarm.ros_distro import Rosdistro
else:
from buildfarm.ros_distro_fuerte import Rosdistro
wet_distro = Rosdistro(distro_name)
# Build the new meta packages
for (v, d) in distro.variants.iteritems():
deb_name = "ros-%s-%s" % (distro_name, debianize_name(v))
# If the metapkg is in the packagelist AND already has the right deps, we leave it:
if deb_name in packagelist:
list_deps = set([
x.split()[0].strip()
for x in packagelist[deb_name]['Depends'].split(',')
])
mp_deps = set([
"ros-%s-%s" % (distro_name, debianize_name(x))
for x in set(d.stack_names) - missing_ok
])
if list_deps == mp_deps:
debug("Metapackage %s already has correct deps" % deb_name)
continue
# Else, we create the new metapkg
mp = create_meta_pkg(packagelist, distro, distro_name, v,
set(d.stack_names) - missing_ok, os_platform,
arch, staging_dir, wet_distro)
if mp:
debs.append(mp)
else:
missing.append(v)
# We should always need to build the special "all" metapackage
#mp = create_meta_pkg(packagelist, distro, distro_name, "all", set(distro.released_stacks.keys()) - missing_ok, os_platform, arch, staging_dir, wet_distro)
#if mp:
# debs.append(mp)
#else:
# missing.append('all')
upload_binary_debs(debs, distro_name, os_platform, arch, repo_fqdn)
if missing:
raise StackBuildFailure("Did not generate all metapkgs: %s." % missing)
expected_branch = 'rpm/%s/%s/' % (rosdistro, fver)
else:
return False
else:
expected_branch = 'rpm/' if not rosdistro else 'rpm/%s/' % (rosdistro,)
for b in repo.get_branches():
if b.name.startswith(expected_branch):
return True
return False
if __name__ == '__main__':
args = parse_options()
sys.stdout.write('loading rosdistro...')
sys.stdout.flush()
rd = Rosdistro(args.rosdistro)
print('got %s' % (rd._rosdistro,))
print('connecting to GitHub:')
gh = github.Github(args.key)
sys.stdout.write('- getting user...')
sys.stdout.flush()
gh_user = gh.get_user()
print('got %s' % gh_user.login)
sys.stdout.write('- getting org...')
sys.stdout.flush()
gh_org = gh.get_organization(args.org)
print('got %s' % gh_org.login)
if not args.repos:
sys.stdout.write('- getting org\'s repo list...')
sys.stdout.flush()
return results
if __name__ == '__main__':
args = parse_options()
print('Loading rosdistro %s' % args.rosdistro)
workspace = args.repo_workspace
if not workspace:
workspace = os.path.join(tempfile.gettempdir(),
'repo-workspace-%s' % args.rosdistro)
if args.rosdistro != 'fuerte':
from buildfarm.ros_distro import Rosdistro
rd = Rosdistro(args.rosdistro)
from buildfarm import dependency_walker
packages = dependency_walker.get_packages(workspace,
rd,
skip_update=args.skip_update)
dependencies = dependency_walker.get_jenkins_dependencies(
args.rosdistro, packages)
# TODO does only work with one build file
build_config = rd._build_files[0].get_target_configuration()
apt_target_repository = build_config['apt_target_repository']
if args.fqdn is None:
fqdn_parts = urlsplit(apt_target_repository)
args.fqdn = fqdn_parts.netloc
if args.arches is None:
args.arches = rd.get_arches()
