def updates_check(self) -> Dict[str, List[Tuple[str, str]]]:
"""
Returns the list of packages to be updated/installed
by a dist-upgrade.
"""
install = []
update = []
remove = []
apt = Cache(memonly=True)
apt.update()
apt.open()
apt.clear()
apt.upgrade(dist_upgrade=True)
for pkg in apt.get_changes():
if pkg.marked_install:
install.append((pkg.name, pkg.candidate.version))
if pkg.marked_upgrade:
update.append((pkg.name, pkg.candidate.version))
if pkg.marked_delete:
remove.append((pkg.name, pkg.installed.version))
return dict(
update=sorted(update),
install=sorted(install),
remove=sorted(remove),
)
def configure_grub(bootdev):
print("Configuring GRUB.")
os.system("mkdir /etc/default/grub.d")
if len(bootdev) == 0:
fipscfg = open("/etc/default/grub.d/99-fips.cfg", "w+")
fipscfg.write(
"GRUB_CMDLINE_LINUX_DEFAULT=\"$GRUB_CMDLINE_LINUX_DEFAULT fips=1\""
)
fipscfg.close()
else:
fipscfg = open("/etc/default/grub.d/99-fips.cfg", "w+")
fipscfg.write(
"GRUB_CMDLINE_LINUX_DEFAULT=\"$GRUB_CMDLINE_LINUX_DEFAULT fips=1 bootdev="
+ bootdev + "\"")
fipscfg.close()
cache = Cache()
cache.open()
kernelnumbers = str(str(
cache["linux-fips"].versions[0]).split("=")[1]).split(".")
kernelversion = str(kernelnumbers[0]) + "." + str(
kernelnumbers[1]) + "." + str(kernelnumbers[2]) + "-" + str(
kernelnumbers[3]) + "-fips"
os.system("mv /etc/default/grub /etc/default/grub.pre-fips")
grubcfg = open("/etc/default/grub.pre-fips", "r")
newgrubcfg = open("/etc/default/grub", "w+")
for line in grubcfg.readlines():
if line.startswith("GRUB_DEFAULT"):
newgrubcfg.write("#" + line)
newgrubcfg.write(
"GRUB_DEFAULT=\"Advanced options for Ubuntu>Ubuntu, with Linux "
+ str(kernelversion) + "\"\n")
else:
newgrubcfg.write(line)
grubcfg.close()
newgrubcfg.close()
def __init__(self,
rfs,
log,
arch,
notifier=None,
norecommend=False,
noauth=True):
sys.stdout = open(log, 'a', buffering=0)
sys.stderr = open(log, 'a', buffering=0)
self.logfile = open(log, 'a', buffering=0)
InChRootObject.__init__(self, rfs)
self.notifier = notifier
config.set("APT::Architecture", arch)
if norecommend:
config.set("APT::Install-Recommends", "1")
else:
config.set("APT::Install-Recommends", "0")
if noauth:
config.set("APT::Get::AllowUnauthenticated", "1")
else:
config.set("APT::Get::AllowUnauthenticated", "0")
self.cache = Cache(progress=ElbeOpProgress())
self.cache.open(progress=ElbeOpProgress())
def start(self):
os.environ["APT_LISTCHANGES_FRONTEND"] = "none"
# Do not suspend during the update process
self.sleep_cookie = inhibit_sleep()
if self.action == self.ACTION_INSTALL:
# Get the packages which should be installed and update
pkgs_install = []
pkgs_upgrade = []
pkgs_remove = []
# Get a fresh cache in case update-manager's is outdated to
# skip operations that already took place
fresh_cache = Cache(rootdir=self.window_main.cache.rootdir)
for pkg in self.window_main.cache:
try:
if pkg.marked_install and \
not fresh_cache[pkg.name].is_installed:
pkgname = pkg.name
if pkg.is_auto_installed:
pkgname += "#auto"
pkgs_install.append(pkgname)
elif (pkg.marked_upgrade and
fresh_cache[pkg.name].is_upgradable):
pkgs_upgrade.append(pkg.name)
elif (pkg.marked_delete and
fresh_cache[pkg.name].is_installed):
pkgs_remove.append(pkg.name)
except KeyError:
# pkg missing from fresh_cache can't be modified
pass
self.commit(pkgs_install, pkgs_upgrade, pkgs_remove)
else:
self.update()
def get_installed_packages():
try:
from apt import Cache
except ImportError as ie:
logging.error(ie)
return ['dnsmasq', 'hostapd'] # assume its installed
return [pkg.name for pkg in Cache() if pkg.is_installed]
def get_initvm_pkglist():
with warnings.catch_warnings():
warnings.filterwarnings("ignore", category=DeprecationWarning)
cache = Cache()
cache.open()
pkglist = [APTPackage(p) for p in cache if p.is_installed]
return pkglist
def __init__(self, db, doc=None, application=None):
super(AppDetailsDebFile, self).__init__(db, doc, application)
if doc:
raise ValueError("doc must be None for deb files")
try:
# for some reason Cache() is much faster than "self._cache._cache"
# on startup
with ExecutionTime("create DebPackage"):
self._deb = DebPackage(self._app.request, Cache())
except:
self._deb = None
self._pkg = None
if not os.path.exists(self._app.request):
self._error = _("Not found")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d does not exist.")) % utf8(
self._app.request)
else:
mimetype = guess_type(self._app.request)
if mimetype[0] != "application/x-debian-package":
self._error = _("Not found")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d is not a software package."
)) % utf8(self._app.request)
else:
# deb files which are corrupt
self._error = _("Internal Error")
self._error_not_found = utf8(
_(u"The file \u201c%s\u201d could not be opened.")
) % utf8(self._app.request)
return
if self.pkgname and self.pkgname != self._app.pkgname:
# this happens when the deb file has a quirky file name
self._app.pkgname = self.pkgname
# load pkg cache
self._pkg = None
if (self._app.pkgname in self._cache
and self._cache[self._app.pkgname].candidate):
self._pkg = self._cache[self._app.pkgname]
# load xapian document
self._doc = None
try:
self._doc = self._db.get_xapian_document(
self._app.appname, self._app.pkgname)
except:
pass
# check deb and set failure state on error
with ExecutionTime("AppDetailsDebFile._deb.check()"):
if not self._deb.check():
self._error = self._deb._failure_string
def security_status(cfg: UAConfig) -> Dict[str, Any]:
"""Returns the status of security updates on a system.
The returned dict has a 'packages' key with a list of all installed
packages which can receive security updates, with or without ESM,
reflecting the availability of the update based on the UA status.
There is also a summary with the UA information and the package counts.
"""
ua_info = get_ua_info(cfg)
summary = {"ua": ua_info} # type: Dict[str, Any]
packages = []
cache = Cache()
installed_packages = [package for package in cache if package.is_installed]
summary["num_installed_packages"] = len(installed_packages)
package_count = defaultdict(int) # type: DefaultDict[str, int]
update_count = defaultdict(int) # type: DefaultDict[str, int]
for package in installed_packages:
package_origin = get_origin_for_package(package)
package_count[package_origin] += 1
security_upgradable_versions = filter_security_updates(installed_packages)
for candidate in security_upgradable_versions:
service_name, origin_site = get_service_name(candidate.origins)
status = get_update_status(service_name, ua_info)
update_count[service_name] += 1
packages.append({
"package": candidate.package.name,
"version": candidate.version,
"service_name": service_name,
"status": status,
"origin": origin_site,
})
summary["num_main_packages"] = package_count["main"]
summary["num_restricted_packages"] = package_count["restricted"]
summary["num_universe_packages"] = package_count["universe"]
summary["num_multiverse_packages"] = package_count["multiverse"]
summary["num_third_party_packages"] = package_count["third-party"]
summary["num_unknown_packages"] = package_count["unknown"]
summary["num_esm_infra_packages"] = package_count["esm-infra"]
summary["num_esm_apps_packages"] = package_count["esm-apps"]
summary["num_esm_infra_updates"] = update_count["esm-infra"]
summary["num_esm_apps_updates"] = update_count["esm-apps"]
summary["num_standard_security_updates"] = update_count[
"standard-security"]
return {"_schema_version": "0.1", "summary": summary, "packages": packages}
def mk_source_cdrom(rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE):
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"), log, cdrom_size)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
dsc = pkgver.fetch_source('/var/cache/elbe/sources',
ElbeAcquireProgress(cb=None),
unpack=False)
repo.includedsc(dsc)
except ValueError as ve:
log.printo("No sources for Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Source for Package " + pkg.name + "-" +
pkgver.version + " could not be downloaded")
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def get_initvm_pkglist ():
with warnings.catch_warnings():
warnings.filterwarnings("ignore",category=DeprecationWarning)
cache = Cache ()
cache.open ()
pkglist = [APTPackage (p) for p in cache if p.is_installed]
try:
eb = APTPackage( cache ['elbe-bootstrap'] )
pkglist.append (eb)
# elbe bootstrap is not installed on pc running elbe
except KeyError:
pass
return pkglist
def main(ignored_packages):
cache = Cache(memonly=True)
not_in_repos = []
ignore = parse_ignore(ignored_packages)
for pkg in cache:
if (pkg.is_installed and not pkg.is_upgradable
and not check_ignored(pkg, ignore)
and not any(pkg.installed.uris)):
not_in_repos.append(pkg)
if not_in_repos:
print('WARNING: packages not in repositories: {}'.format(' '.join(
pkg.name for pkg in not_in_repos)))
exit(1)
print('OK: All packages are found in repositories')
exit(0)
def get_pt_installed_software(self):
try:
from apt import Cache
except ModuleNotFoundError:
# probably running in virtualenv. python3-pip is not pip installable
print("Error: couldn't find python APT library. Skipping...")
return []
regex = "^pt-|-pt-|pitop|pi-top"
apt_cache = Cache()
pt_packages = []
for pkg in apt_cache:
match = search(regex, pkg.name)
if apt_cache[pkg.name].is_installed and match:
pt_packages.append((pkg.shortname, pkg.installed.version))
return pt_packages
def __init__( self, rfs, logpath, arch, notifier=None, norecommend = False, noauth = True ):
self.log = ASCIIDocLog(logpath)
self.notifier = notifier
InChRootObject.__init__(self, rfs)
config.set ("APT::Architecture", arch)
if norecommend:
config.set ("APT::Install-Recommends", "1")
else:
config.set ("APT::Install-Recommends", "0")
if noauth:
config.set ("APT::Get::AllowUnauthenticated", "1")
else:
config.set ("APT::Get::AllowUnauthenticated", "0")
self.cache = Cache()
self.cache.open()
def prune(opt: Namespace) -> None:
cache = Cache()
cur = {PREFIX + uname()[2] + suffix for suffix in {"", "-signed"}}
top = {
"univention-kernel-image",
PREFIX + "amd64",
PREFIX + "rt-amd64",
PREFIX + "686-pae",
PREFIX + "686",
PREFIX + "rt-686-pae",
}
meta = [
pkg.installed for pkg in (cache[pkg] for pkg in top if pkg in cache)
if pkg.is_installed
]
if opt.verbose:
print("Installed kernel meta packages:\n %s" %
("\n ".join(sorted(str(pkg) for pkg in meta)), ))
keep = ({
dep.name
for pkg in meta for alt in pkg.dependencies
for dep in alt if dep.name.startswith(PREFIX)
}
| cur
| top)
if opt.verbose:
print("Exception list for kernel packages:\n %s" %
("\n ".join(sorted(keep)), ))
cache.clear()
for pkg in cache:
if pkg.name.startswith(PREFIX) and pkg.is_installed:
if pkg.name not in keep:
if opt.verbose:
print("Purging kernel package: %s" % (pkg.name, ))
if not opt.dry_run:
pkg.mark_delete(purge=True)
else:
if opt.verbose:
print("Keeping kernel package: %s" % (pkg.name, ))
cache.commit()
def restart_server():
# This function will restart NginX, MariaDB, PHP5-FPM if installed.
cache = Cache()
if cache['nginx'].is_installed:
system("sudo service nginx restart")
if cache['mariadb-server'].is_installed:
system("sudo service mysql restart")
if cache['php5-fpm'].is_installed:
system("sudo service php5-fpm restart")
else:
print("PHP isn't installed. So, it hasn't been started.")
else:
print("MariaDB isn't installed. So, it hasn't been started.")
elif cache['mysql'].is_installed:
system("sudo service mysql restart")
if cache['php5-fpm'].is_installed:
system("sudo service php5-fpm restart")
else:
print("PHP isn't installed. So, it hasn't been started.")
else:
system("sudo service php5-fpm restart")
def checkInstall(pkgName, action, files):
x, y, f_list, apt_cache = 0, 0, files.split(','), Cache()
try:
for e in pkgName.split('_'):
if apt_cache[e].is_installed:
x, y = x + 1, y + 1
except:
pass
for i in f_list:
if isfile(i) or isdir(i):
x = x + 1
if '--install' in action and x != 0:
print "\033[0;31mERROR: {} is already installed.\033[0m".format(
pkgName)
return False
elif '--remove' in action and x != len(f_list) + y:
print "\033[0;31mERROR: {} is not installed.\033[0m\n".format(pkgName)
return False
print "%s\t%34s\033[0;32m %s \033[0m]" % (
" * Check if {} is installed ".format(pkgName), "[", "OK")
return True
def main(ignored_packages):
cache = Cache(memonly=True)
not_in_repos = []
ignore = parse_ignore(ignored_packages)
for pkg in cache:
if (pkg.is_installed and not pkg.is_upgradable
and not check_ignored(pkg, ignore)
and not any(pkg.installed.uris)):
not_in_repos.append(pkg)
if not_in_repos:
pkgs = ' '.join([pkg.name for pkg in not_in_repos])
msg = u'WARNING: {0} packages have no candidate in repositorys!' \
' | {1}'.format(len(not_in_repos), pkgs)
sig = 1
else:
msg = 'OK: All packages are found in repositorys'
sig = 0
print(msg)
exit(sig)
def check_for_installed_packages(config, provider):
"""
Checks if the required host packages have been installed
:param dict config: The config provided by get_config()
:param str provider: The provider to check the required host packages for
:return bool: True if all required packages have been installed, False otherwise
"""
logger.debug(
"Checking if all required host packages have been install for provider {}"
.format(provider))
all_installed = True
cache = Cache()
if "apt" not in modules:
logger.warning(
"Skipping host package rquirements check, apt module missing")
else:
for package in config["providers"][provider]["required_host_packages"]:
if not cache[package].is_installed:
logger.error(
"Required host package {} for provider {} is not installed"
.format(package, provider))
all_installed = False
return all_installed
def train_model(self, pkgs_list, axi, save_files=True):
cache = Cache()
ml_data = MachineLearningData()
pkgs_description, pkg_classification = self.prepare_data(
pkgs_list, axi, cache, ml_data)
pkg_features = self.vectorizer.fit_transform(pkgs_description)
features_array = pkg_features.toarray()
terms, debtags = self.get_used_terms_and_debtags(
self.vectorizer.get_feature_names())
self.classifier = GaussianNB()
self.classifier.fit(features_array, pkg_classification)
path = BagOfWords.BAG_OF_WORDS_PKGS_CLASSIFICATION
if save_files:
self.save_features(terms, BagOfWords.BAG_OF_WORDS_TERMS)
self.save_features(debtags, BagOfWords.BAG_OF_WORDS_DEBTAGS)
self.save_pkgs_features(path, pkgs_list, features_array,
pkg_classification)
return BagOfWords.CREATED_MODEL
def __init__(self,
rfs,
arch,
notifier=None,
norecommend=False,
noauth=True):
# pylint: disable=too-many-arguments
InChRootObject.__init__(self, rfs)
self.notifier = notifier
config.set("APT::Architecture", arch)
if norecommend:
config.set("APT::Install-Recommends", "0")
else:
config.set("APT::Install-Recommends", "1")
if noauth:
config.set("APT::Get::AllowUnauthenticated", "1")
else:
config.set("APT::Get::AllowUnauthenticated", "0")
self.cache = Cache(progress=ElbeOpProgress())
self.cache.open(progress=ElbeOpProgress())
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description='Lists available binary packages and versions which are'
'needed to satisfy rosdep keys for ROS packages in the workspace')
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_output_dir(parser)
add_argument_package_selection_args(parser)
add_argument_skip_rosdep_keys(parser)
parser.add_argument('--package-root',
nargs='+',
help='The path to the directory containing packages')
args = parser.parse_args(argv)
workspace_root = args.package_root[-1]
os.chdir(workspace_root)
with Scope('SUBSECTION', 'mark packages with IGNORE files'):
all_packages = locate_packages(workspace_root)
selected_packages = all_packages
if args.package_selection_args:
print('Using package selection arguments:',
args.package_selection_args)
selected_packages = locate_packages(
workspace_root, extra_args=args.package_selection_args)
to_ignore = all_packages.keys() - selected_packages.keys()
print('Ignoring %d packages' % len(to_ignore))
for package in sorted(to_ignore):
print('-', package)
package_root = all_packages[package]
Path(package_root, 'COLCON_IGNORE').touch()
print('There are %d packages which meet selection criteria' %
len(selected_packages))
with Scope('SUBSECTION', 'Enumerating packages needed to build'):
# find all of the underlay packages
underlay_pkgs = {}
all_underlay_pkg_names = set()
for package_root in args.package_root[0:-1]:
print("Crawling for packages in '%s'" % package_root)
underlay_pkgs.update(find_packages(package_root))
# Check for a colcon index for non-ROS package detection
colcon_index = os.path.join(package_root, 'colcon-core',
'packages')
try:
all_underlay_pkg_names.update(os.listdir(colcon_index))
except FileNotFoundError:
pass
underlay_pkg_names = [pkg.name for pkg in underlay_pkgs.values()]
print('Found the following ROS underlay packages:')
for pkg_name in sorted(underlay_pkg_names):
print(' -', pkg_name)
# get direct build dependencies
package_root = args.package_root[-1]
print("Crawling for packages in '%s'" % package_root)
pkgs = find_packages(package_root)
pkg_names = [pkg.name for pkg in pkgs.values()]
print('Found the following ROS packages:')
for pkg_name in sorted(pkg_names):
print(' -', pkg_name)
# get build dependencies and map them to binary packages
all_pkgs = set(pkgs.values()).union(underlay_pkgs.values())
for pkg in all_pkgs:
pkg.evaluate_conditions(os.environ)
for pkg in all_pkgs:
for group_depend in pkg.group_depends:
if group_depend.evaluated_condition:
group_depend.extract_group_members(all_pkgs)
dependency_keys_build = get_dependencies(
all_pkgs, 'build', _get_build_and_recursive_run_dependencies,
pkgs.values())
dependency_keys_test = get_dependencies(
all_pkgs, 'run and test', _get_test_and_recursive_run_dependencies,
pkgs.values())
if args.skip_rosdep_keys:
dependency_keys_build.difference_update(args.skip_rosdep_keys)
dependency_keys_test.difference_update(args.skip_rosdep_keys)
# remove all non-ROS packages and packages which are present but
# specifically ignored
every_package_name = all_packages.keys() | all_underlay_pkg_names
dependency_keys_build -= every_package_name
dependency_keys_test -= every_package_name
context = initialize_resolver(args.rosdistro_name, args.os_name,
args.os_code_name)
os_pkg_names_build = resolve_names(dependency_keys_build, **context)
os_pkg_names_test = resolve_names(dependency_keys_test, **context)
os_pkg_names_test -= os_pkg_names_build
with Scope('SUBSECTION', 'Resolving packages versions using apt cache'):
apt_cache = Cache()
os_pkg_versions = get_binary_package_versions(
apt_cache, os_pkg_names_build | os_pkg_names_test)
with open(os.path.join(args.output_dir, 'install_list_build.txt'),
'w') as out_file:
for package in sorted(os_pkg_names_build):
out_file.write('%s=%s\n' % (package, os_pkg_versions[package]))
with open(os.path.join(args.output_dir, 'install_list_test.txt'),
'w') as out_file:
for package in sorted(os_pkg_names_test):
out_file.write('%s=%s\n' % (package, os_pkg_versions[package]))
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for building the binarydeb")
add_argument_rosdistro_index_url(parser)
add_argument_rosdistro_name(parser)
add_argument_package_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_distribution_repository_urls(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_binarydeb_dir(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
args = parser.parse_args(argv)
debian_package_name = get_debian_package_name(
args.rosdistro_name, args.package_name)
# get expected package version from rosdistro
index = get_index(args.rosdistro_index_url)
dist_file = get_distribution_file(index, args.rosdistro_name)
assert args.package_name in dist_file.release_packages
pkg = dist_file.release_packages[args.package_name]
repo = dist_file.repositories[pkg.repository_name]
package_version = repo.release_repository.version
debian_package_version = package_version
# build_binarydeb dependencies
debian_pkg_names = ['apt-src']
# add build dependencies from .dsc file
dsc_file = get_dsc_file(
args.binarydeb_dir, debian_package_name, debian_package_version)
debian_pkg_names += sorted(get_build_depends(dsc_file))
# get versions for build dependencies
apt_cache = Cache()
debian_pkg_versions = get_binary_package_versions(
apt_cache, debian_pkg_names)
# generate Dockerfile
data = {
'os_name': args.os_name,
'os_code_name': args.os_code_name,
'arch': args.arch,
'uid': get_user_id(),
'distribution_repository_urls': args.distribution_repository_urls,
'distribution_repository_keys': get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'build_environment_variables': args.env_vars,
'dependencies': debian_pkg_names,
'dependency_versions': debian_pkg_versions,
'install_lists': [],
'rosdistro_name': args.rosdistro_name,
'package_name': args.package_name,
'binarydeb_dir': args.binarydeb_dir,
}
create_dockerfile(
'release/binarydeb_task.Dockerfile.em', data, args.dockerfile_dir)
# output hints about necessary volumes to mount
ros_buildfarm_basepath = os.path.normpath(
os.path.join(os.path.dirname(__file__), '..', '..'))
print('Mount the following volumes when running the container:')
print(' -v %s:/tmp/ros_buildfarm:ro' % ros_buildfarm_basepath)
print(' -v %s:/tmp/binarydeb' % args.binarydeb_dir)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the CI job")
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_distribution_repository_urls(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
add_argument_package_selection_args(parser)
add_argument_repos_file_urls(parser)
add_argument_repository_names(parser, optional=True)
add_argument_skip_rosdep_keys(parser)
add_argument_test_branch(parser)
parser.add_argument('--workspace-root',
nargs='+',
help='The root path of the workspace to compile')
args = parser.parse_args(argv)
assert args.repos_file_urls or args.repository_names
debian_pkg_names = [
'git',
'python3-apt',
'python3-colcon-metadata',
'python3-colcon-package-information',
'python3-colcon-package-selection',
'python3-colcon-recursive-crawl',
'python3-colcon-ros',
'python3-rosdep',
'python3-vcstool',
]
# get versions for build dependencies
apt_cache = Cache()
debian_pkg_versions = get_binary_package_versions(apt_cache,
debian_pkg_names)
# generate Dockerfile
data = {
'os_name':
args.os_name,
'os_code_name':
args.os_code_name,
'arch':
args.arch,
'distribution_repository_urls':
args.distribution_repository_urls,
'distribution_repository_keys':
get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'rosdistro_name':
args.rosdistro_name,
'custom_rosdep_urls': [],
'uid':
get_user_id(),
'build_environment_variables':
['%s=%s' % key_value for key_value in args.env_vars.items()],
'dependencies':
debian_pkg_names,
'dependency_versions':
debian_pkg_versions,
'repos_file_urls':
args.repos_file_urls,
'repository_names':
args.repository_names,
'test_branch':
args.test_branch,
'skip_rosdep_keys':
args.skip_rosdep_keys,
'package_selection_args':
args.package_selection_args,
'workspace_root':
args.workspace_root,
}
create_dockerfile('ci/create_workspace.Dockerfile.em', data,
args.dockerfile_dir)
#!/usr/bin/python
from apt import Cache
import argparse
from collections import namedtuple
import re
import subprocess
import sys
APT_CACHE = Cache()
DEPENDS = ('Depends', 'PreDepends', 'Recommends')
DEPENDENCY_RE = re.compile(
' +(?P<type>[^:]+): (?P<pkg_name>\S+)(?: (?P<version>.+))')
DOTTY_STYLE = {
'Reverse Recommends': '[style=dotted color="#999999"]',
}
PackageDependency = namedtuple('PackageDependency',
('name', 'type', 'version'))
TreeNode = namedtuple('TreeNode', ('name', 'children'))
class Error(Exception):
"""Error ocurred."""
class DependencyTree(object):
def __init__(self, package):
self.package_ = package
self.dependencies_ = None
self.tree_ = None
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the CI job")
# Positional
add_argument_rosdistro_name(parser)
add_argument_os_name(parser)
add_argument_os_code_name(parser)
add_argument_arch(parser)
add_argument_build_tool(parser, required=True)
add_argument_build_tool_args(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_distribution_repository_urls(parser)
add_argument_dockerfile_dir(parser)
add_argument_env_vars(parser)
add_argument_install_packages(parser)
add_argument_ros_version(parser)
add_argument_testing(parser)
parser.add_argument(
'--workspace-root', nargs='*',
action=check_len_action(1, 2),
help='The root path of the workspace to compile')
args = parser.parse_args(argv)
apt_cache = Cache()
debian_pkg_names = set(['build-essential'])
debian_pkg_names.update(args.install_packages)
if args.build_tool == 'colcon':
debian_pkg_names.update([
'python3-catkin-pkg-modules',
'python3-colcon-output',
'python3-colcon-parallel-executor',
'python3-colcon-ros',
'python3-colcon-test-result',
'python3-rosdistro-modules',
])
print('Always install the following generic dependencies:')
for debian_pkg_name in sorted(debian_pkg_names):
print(' -', debian_pkg_name)
install_list = 'install_list.txt'
write_install_list(
os.path.join(args.dockerfile_dir, install_list),
debian_pkg_names, apt_cache)
install_lists = [install_list, 'install_list_build.txt']
if args.testing:
install_lists.append('install_list_test.txt')
# generate Dockerfile
data = {
'os_name': args.os_name,
'os_code_name': args.os_code_name,
'arch': args.arch,
'distribution_repository_urls': args.distribution_repository_urls,
'distribution_repository_keys': get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'rosdistro_name': args.rosdistro_name,
'uid': get_user_id(),
'build_tool': args.build_tool,
'build_tool_args': args.build_tool_args,
'ros_version': args.ros_version,
'build_environment_variables': args.env_vars,
'install_lists': install_lists,
'dependencies': [],
'dependency_versions': [],
'testing': args.testing,
'prerelease_overlay': len(args.workspace_root) > 1,
}
create_dockerfile(
'devel/devel_task.Dockerfile.em', data, args.dockerfile_dir)
# output hints about necessary volumes to mount
ros_buildfarm_basepath = os.path.normpath(
os.path.join(os.path.dirname(__file__), '..', '..'))
print('Mount the following volumes when running the container:')
print(' -v %s:/tmp/ros_buildfarm:ro' % ros_buildfarm_basepath)
if len(args.workspace_root) == 1:
print(' -v %s:/tmp/ws' % args.workspace_root[0])
else:
for i, workspace_root in enumerate(args.workspace_root[0:-1]):
print(' -v %s:/tmp/ws%s' % (workspace_root, i or ''))
print(' -v %s:/tmp/ws_overlay' % args.workspace_root[-1])
def is_installed(package):
cache = Cache()
return cache[package].is_installed if package in cache else False
def run_command(argv):
# TODO - Set threshold and remove pylint directives
#
# We might want to make the threshold higher for certain
# files/directories or just globaly.
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
oparser = OptionParser(
usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>")
oparser.add_option("-b",
"--binrepo",
dest="binrepo",
default="/var/cache/elbe/initvm-bin-repo",
help="directory where the bin repo should reside")
oparser.add_option("-s",
"--srcrepo",
dest="srcrepo",
default="/var/cache/elbe/initvm-src-repo",
help="directory where the src repo should reside")
oparser.add_option("--skip-validation",
action="store_true",
dest="skip_validation",
default=False,
help="Skip xml schema validation")
oparser.add_option("--cdrom-mount-path",
dest="cdrom_path",
help="path where cdrom is mounted")
oparser.add_option("--cdrom-device",
dest="cdrom_device",
help="cdrom device, in case it has to be mounted")
oparser.add_option("--apt-archive",
dest="archive",
default="/var/cache/elbe/binaries/main",
help="path where binary packages are downloaded to.")
oparser.add_option("--src-archive",
dest="srcarchive",
default="/var/cache/elbe/sources",
help="path where src packages are downloaded to.")
oparser.add_option("--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip downloading Source Packages")
oparser.add_option("--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip downloading binary packages")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print("wrong number of arguments")
oparser.print_help()
sys.exit(20)
try:
xml = ElbeXML(args[0], skip_validate=opt.skip_validation)
except ValidationError as e:
print(str(e))
print("xml validation failed. Bailing out")
sys.exit(20)
with elbe_logging({"streams": sys.stdout}):
if opt.cdrom_path:
if opt.cdrom_device:
do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path))
# a cdrom build is identified by the cdrom option
# the xml file that is copied into the initvm
# by the initrd does not have the cdrom tags setup.
mirror = "file://%s" % opt.cdrom_path
else:
mirror = xml.get_initvm_primary_mirror(opt.cdrom_path)
init_codename = xml.get_initvm_codename()
# Binary Repo
#
repo = CdromInitRepo(init_codename, opt.binrepo, mirror)
hostfs.mkdir_p(opt.archive)
if opt.build_bin:
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
deb = fetch_binary(pkgver, opt.archive,
ElbeAcquireProgress(cb=None))
repo.includedeb(deb, 'main')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
# Source Repo
#
repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0,
mirror)
hostfs.mkdir_p(opt.srcarchive)
# a cdrom build does not have sources
# skip adding packages to the source repo
#
# FIXME: we need a way to add source cdroms later on
if opt.cdrom_path:
opt.build_sources = False
if opt.build_sources:
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
p = cache[pkg.name]
pkgver = p.installed
dsc = pkgver.fetch_source(opt.srcarchive,
ElbeAcquireProgress(cb=None),
unpack=False)
repo.include_init_dsc(dsc, 'initvm')
except ValueError:
logging.exception('No package "%s"', pkg_id)
except FetchError:
logging.exception(
'Package "%s-%s" could not be downloaded', pkg.name,
pkgver.version)
except TypeError:
logging.exception('Package "%s" missing name or version',
pkg_id)
repo.finalize()
if opt.cdrom_device:
do('umount "%s"' % opt.cdrom_device)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the doc job")
add_argument_config_url(parser)
parser.add_argument(
'--rosdistro-name',
required=True,
help='The name of the ROS distro to identify the setup file to be '
'sourced')
add_argument_build_name(parser, 'doc')
parser.add_argument('--workspace-root',
required=True,
help='The root path of the workspace to compile')
parser.add_argument('--rosdoc-lite-dir',
required=True,
help='The root path of the rosdoc_lite repository')
parser.add_argument('--catkin-sphinx-dir',
required=True,
help='The root path of the catkin-sphinx repository')
parser.add_argument('--rosdoc-index-dir',
required=True,
help='The root path of the rosdoc_index folder')
add_argument_repository_name(parser)
parser.add_argument('--os-name',
required=True,
help="The OS name (e.g. 'ubuntu')")
parser.add_argument('--os-code-name',
required=True,
help="The OS code name (e.g. 'xenial')")
parser.add_argument('--arch',
required=True,
help="The architecture (e.g. 'amd64')")
add_argument_build_tool(parser, required=True)
add_argument_vcs_information(parser)
add_argument_distribution_repository_urls(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_force(parser)
add_argument_output_dir(parser, required=True)
add_argument_dockerfile_dir(parser)
args = parser.parse_args(argv)
config = get_config_index(args.config_url)
index = get_index(config.rosdistro_index_url)
condition_context = get_package_condition_context(index,
args.rosdistro_name)
with Scope('SUBSECTION', 'packages'):
# find packages in workspace
source_space = os.path.join(args.workspace_root, 'src')
print("Crawling for packages in workspace '%s'" % source_space)
pkgs = find_packages(source_space)
for pkg in pkgs.values():
pkg.evaluate_conditions(condition_context)
pkg_names = [pkg.name for pkg in pkgs.values()]
print('Found the following packages:')
for pkg_name in sorted(pkg_names):
print(' -', pkg_name)
maintainer_emails = set([])
for pkg in pkgs.values():
for m in pkg.maintainers:
maintainer_emails.add(m.email)
if maintainer_emails:
print('Package maintainer emails: %s' %
' '.join(sorted(maintainer_emails)))
rosdoc_index = RosdocIndex(
[os.path.join(args.rosdoc_index_dir, args.rosdistro_name)])
vcs_type, vcs_version, vcs_url = args.vcs_info.split(' ', 2)
with Scope('SUBSECTION', 'determine need to run documentation generation'):
# compare hashes to determine if documentation needs to be regenerated
current_hashes = {}
current_hashes['ros_buildfarm'] = 2 # increase to retrigger doc jobs
current_hashes['rosdoc_lite'] = get_git_hash(args.rosdoc_lite_dir)
current_hashes['catkin-sphinx'] = get_git_hash(args.catkin_sphinx_dir)
repo_dir = os.path.join(args.workspace_root, 'src',
args.repository_name)
current_hashes[args.repository_name] = get_hash(repo_dir)
print('Current repository hashes: %s' % current_hashes)
tag_index_hashes = rosdoc_index.hashes.get(args.repository_name, {})
print('Stored repository hashes: %s' % tag_index_hashes)
skip_doc_generation = current_hashes == tag_index_hashes
if skip_doc_generation:
print('No changes to the source repository or any tooling repository')
if not args.force:
print('Skipping generation of documentation')
# create stamp files
print('Creating marker files to identify that documentation is ' +
'up-to-date')
create_stamp_files(pkg_names, os.path.join(args.output_dir, 'api'))
# check if any entry needs to be updated
print('Creating update manifest.yaml files')
for pkg_name in pkg_names:
# update manifest.yaml files
current_manifest_yaml_file = os.path.join(
args.rosdoc_index_dir, args.rosdistro_name, 'api',
pkg_name, 'manifest.yaml')
if not os.path.exists(current_manifest_yaml_file):
print('- %s: skipping no manifest.yaml yet' % pkg_name)
continue
with open(current_manifest_yaml_file, 'r') as h:
remote_data = yaml.safe_load(h)
data = copy.deepcopy(remote_data)
data['vcs'] = vcs_type
data['vcs_uri'] = vcs_url
data['vcs_version'] = vcs_version
data['depends_on'] = sorted(
rosdoc_index.reverse_deps.get(pkg_name, []))
if data == remote_data:
print('- %s: skipping same data' % pkg_name)
continue
# write manifest.yaml if it has changes
print('- %s: api/%s/manifest.yaml' % (pkg_name, pkg_name))
dst = os.path.join(args.output_dir, 'api', pkg_name,
'manifest.yaml')
dst_dir = os.path.dirname(dst)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
with open(dst, 'w') as h:
yaml.dump(data, h, default_flow_style=False)
return 0
print("But job was started with the 'force' parameter set")
else:
print('The source repository and/or a tooling repository has changed')
print('Running generation of documentation')
rosdoc_index.hashes[args.repository_name] = current_hashes
rosdoc_index.write_modified_data(args.output_dir, ['hashes'])
# create stamp files
print('Creating marker files to identify that documentation is ' +
'up-to-date')
create_stamp_files(pkg_names, os.path.join(args.output_dir, 'api_rosdoc'))
dist_file = get_distribution_file(index, args.rosdistro_name)
assert args.repository_name in dist_file.repositories
valid_package_names = \
set(pkg_names) | set(dist_file.release_packages.keys())
# update package deps and metapackage deps
with Scope('SUBSECTION', 'updated rosdoc_index information'):
for pkg in pkgs.values():
print("Updating dependendencies for package '%s'" % pkg.name)
depends = _get_build_run_doc_dependencies(pkg)
ros_dependency_names = sorted(
set([d.name for d in depends
if d.name in valid_package_names]))
rosdoc_index.set_forward_deps(pkg.name, ros_dependency_names)
if pkg.is_metapackage():
print("Updating dependendencies for metapackage '%s'" %
pkg.name)
depends = _get_run_dependencies(pkg)
ros_dependency_names = sorted(
set([
d.name for d in depends
if d.name in valid_package_names
]))
else:
ros_dependency_names = None
rosdoc_index.set_metapackage_deps(pkg.name, ros_dependency_names)
rosdoc_index.write_modified_data(args.output_dir,
['deps', 'metapackage_deps'])
# generate changelog html from rst
package_names_with_changelogs = set([])
with Scope('SUBSECTION', 'generate changelog html from rst'):
for pkg_path, pkg in pkgs.items():
abs_pkg_path = os.path.join(source_space, pkg_path)
assert os.path.exists(os.path.join(abs_pkg_path, 'package.xml'))
changelog_file = os.path.join(abs_pkg_path, 'CHANGELOG.rst')
if os.path.exists(changelog_file):
print(("Package '%s' contains a CHANGELOG.rst, generating " +
"html") % pkg.name)
package_names_with_changelogs.add(pkg.name)
with open(changelog_file, 'r') as h:
rst_code = h.read()
from docutils.core import publish_string
html_code = publish_string(rst_code, writer_name='html')
html_code = html_code.decode()
# strip system message from html output
open_tag = re.escape('<div class="first system-message">')
close_tag = re.escape('</div>')
pattern = '(' + open_tag + '.+?' + close_tag + ')'
html_code = re.sub(pattern, '', html_code, flags=re.DOTALL)
pkg_changelog_doc_path = os.path.join(args.output_dir,
'changelogs', pkg.name)
os.makedirs(pkg_changelog_doc_path)
with open(
os.path.join(pkg_changelog_doc_path, 'changelog.html'),
'w') as h:
h.write(html_code)
ordered_pkg_tuples = topological_order_packages(pkgs)
# create rosdoc tag list and location files
with Scope('SUBSECTION', 'create rosdoc tag list and location files'):
rosdoc_config_files = {}
for pkg_path, pkg in pkgs.items():
abs_pkg_path = os.path.join(source_space, pkg_path)
rosdoc_exports = [
e.attributes['content'] for e in pkg.exports
if e.tagname == 'rosdoc' and 'content' in e.attributes
]
prefix = '${prefix}'
rosdoc_config_file = rosdoc_exports[-1] \
if rosdoc_exports else '%s/rosdoc.yaml' % prefix
rosdoc_config_file = rosdoc_config_file.replace(
prefix, abs_pkg_path)
if os.path.isfile(rosdoc_config_file):
rosdoc_config_files[pkg.name] = rosdoc_config_file
for _, pkg in ordered_pkg_tuples:
dst = os.path.join(args.output_dir, 'rosdoc_tags',
'%s.yaml' % pkg.name)
print("Generating rosdoc tag list file for package '%s'" %
pkg.name)
dep_names = rosdoc_index.get_recursive_dependencies(pkg.name)
# make sure that we don't pass our own tagfile to ourself
# bad things happen when we do this
assert pkg.name not in dep_names
locations = []
for dep_name in sorted(dep_names):
if dep_name not in rosdoc_index.locations:
print("- skipping not existing location file of " +
"dependency '%s'" % dep_name)
continue
print("- including location files of dependency '%s'" %
dep_name)
dep_locations = rosdoc_index.locations[dep_name]
if dep_locations:
for dep_location in dep_locations:
assert dep_location['package'] == dep_name
# update tag information to point to local location
location = copy.deepcopy(dep_location)
if not location['location'].startswith('file://'):
location['location'] = 'file://%s' % os.path.join(
args.rosdoc_index_dir, location['location'])
locations.append(location)
dst_dir = os.path.dirname(dst)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
with open(dst, 'w') as h:
yaml.dump(locations, h)
print("Creating location file for package '%s'" % pkg.name)
data = {
'docs_url':
'../../../api/%s/html' % pkg.name,
'location':
'file://%s' %
os.path.join(args.output_dir, 'symbols', '%s.tag' % pkg.name),
'package':
pkg.name,
}
# fetch generator specific output folders from rosdoc_lite
if pkg.name in rosdoc_config_files:
output_folders = get_generator_output_folders(
rosdoc_config_files[pkg.name], pkg.name)
if 'doxygen' in output_folders:
data['docs_url'] += '/' + output_folders['doxygen']
rosdoc_index.locations[pkg.name] = [data]
# do not write these local locations
# used to determine all source and release jobs
source_build_files = get_source_build_files(config, args.rosdistro_name)
release_build_files = get_release_build_files(config, args.rosdistro_name)
# TODO this should reuse the logic from the job generation
used_source_build_names = []
for source_build_name, build_file in source_build_files.items():
repo_names = build_file.filter_repositories([args.repository_name])
if not repo_names:
continue
matching_dist_file = get_distribution_file_matching_build_file(
index, args.rosdistro_name, build_file)
repo = matching_dist_file.repositories[args.repository_name]
if not repo.source_repository:
continue
if not repo.source_repository.version:
continue
if build_file.test_commits_force is False:
continue
elif repo.source_repository.test_commits is False:
continue
elif repo.source_repository.test_commits is None and \
not build_file.test_commits_default:
continue
used_source_build_names.append(source_build_name)
doc_build_files = get_doc_build_files(config, args.rosdistro_name)
doc_build_file = doc_build_files[args.doc_build_name]
# create manifest.yaml files from repository / package meta information
# will be merged with the manifest.yaml file generated by rosdoc_lite later
repository = dist_file.repositories[args.repository_name]
with Scope('SUBSECTION', 'create manifest.yaml files'):
for pkg in pkgs.values():
data = {}
data['vcs'] = vcs_type
data['vcs_uri'] = vcs_url
data['vcs_version'] = vcs_version
data['repo_name'] = args.repository_name
data['timestamp'] = time.time()
data['depends'] = sorted(
rosdoc_index.forward_deps.get(pkg.name, []))
data['depends_on'] = sorted(
rosdoc_index.reverse_deps.get(pkg.name, []))
if pkg.name in rosdoc_index.metapackage_index:
data['metapackages'] = rosdoc_index.metapackage_index[pkg.name]
if pkg.name in rosdoc_index.metapackage_deps:
data['packages'] = rosdoc_index.metapackage_deps[pkg.name]
if pkg.name in package_names_with_changelogs:
data['has_changelog_rst'] = True
data['api_documentation'] = '%s/%s/api/%s/html' % \
(doc_build_file.canonical_base_url, args.rosdistro_name, pkg.name)
pkg_status = None
pkg_status_description = None
# package level status information
if pkg.name in repository.status_per_package:
pkg_status_data = repository.status_per_package[pkg.name]
pkg_status = pkg_status_data.get('status', None)
pkg_status_description = pkg_status_data.get(
'status_description', None)
# repository level status information
if pkg_status is None:
pkg_status = repository.status
if pkg_status_description is None:
pkg_status_description = repository.status_description
if pkg_status is not None:
data['maintainer_status'] = pkg_status
if pkg_status_description is not None:
data['maintainer_status_description'] = pkg_status_description
# add doc job url
data['doc_job'] = get_doc_job_url(config.jenkins_url,
args.rosdistro_name,
args.doc_build_name,
args.repository_name,
args.os_name, args.os_code_name,
args.arch)
# add devel job urls
build_files = {}
for build_name in used_source_build_names:
build_files[build_name] = source_build_files[build_name]
devel_job_urls = get_devel_job_urls(config.jenkins_url,
build_files,
args.rosdistro_name,
args.repository_name)
if devel_job_urls:
data['devel_jobs'] = devel_job_urls
# TODO this should reuse the logic from the job generation
used_release_build_names = []
for release_build_name, build_file in release_build_files.items():
filtered_pkg_names = build_file.filter_packages([pkg.name])
if not filtered_pkg_names:
continue
matching_dist_file = get_distribution_file_matching_build_file(
index, args.rosdistro_name, build_file)
repo = matching_dist_file.repositories[args.repository_name]
if not repo.release_repository:
continue
if not repo.release_repository.version:
continue
used_release_build_names.append(release_build_name)
# add release job urls
build_files = {}
for build_name in used_release_build_names:
build_files[build_name] = release_build_files[build_name]
release_job_urls = get_release_job_urls(config.jenkins_url,
build_files,
args.rosdistro_name,
pkg.name)
if release_job_urls:
data['release_jobs'] = release_job_urls
# write manifest.yaml
dst = os.path.join(args.output_dir, 'manifests', pkg.name,
'manifest.yaml')
dst_dir = os.path.dirname(dst)
if not os.path.exists(dst_dir):
os.makedirs(dst_dir)
with open(dst, 'w') as h:
yaml.dump(data, h)
# overwrite CMakeLists.txt files of each package
with Scope('SUBSECTION',
'overwrite CMakeLists.txt files to only generate messages'):
for pkg_path, pkg in pkgs.items():
abs_pkg_path = os.path.join(source_space, pkg_path)
build_types = [
e.content for e in pkg.exports if e.tagname == 'build_type'
]
build_type_cmake = build_types and build_types[0] == 'cmake'
data = {
'package_name': pkg.name,
'build_type_cmake': build_type_cmake,
}
content = expand_template('doc/CMakeLists.txt.em', data)
print("Generating 'CMakeLists.txt' for package '%s'" % pkg.name)
cmakelist_file = os.path.join(abs_pkg_path, 'CMakeLists.txt')
with open(cmakelist_file, 'w') as h:
h.write(content)
with Scope('SUBSECTION', 'determine dependencies and generate Dockerfile'):
# initialize rosdep view
context = initialize_resolver(args.rosdistro_name, args.os_name,
args.os_code_name)
apt_cache = Cache()
debian_pkg_names = [
'build-essential',
'openssh-client',
'python3',
'python3-yaml',
'rsync',
# the following are required by rosdoc_lite
'doxygen',
# since catkin is not a run dependency but provides the setup files
get_os_package_name(args.rosdistro_name, 'catkin'),
# rosdoc_lite does not work without genmsg being importable
get_os_package_name(args.rosdistro_name, 'genmsg'),
]
if '3' == str(condition_context['ROS_PYTHON_VERSION']):
# the following are required by rosdoc_lite
debian_pkg_names.extend([
'python3-catkin-pkg-modules', 'python3-kitchen',
'python3-rospkg-modules', 'python3-sphinx', 'python3-yaml'
])
else:
if '2' != str(condition_context['ROS_PYTHON_VERSION']):
print('Unknown python version, using Python 2',
condition_context)
# the following are required by rosdoc_lite
debian_pkg_names.extend([
'python-catkin-pkg-modules', 'python-epydoc', 'python-kitchen',
'python-rospkg', 'python-sphinx', 'python-yaml'
])
if args.build_tool == 'colcon':
debian_pkg_names.append('python3-colcon-ros')
if 'actionlib_msgs' in pkg_names:
# to document actions in other packages in the same repository
debian_pkg_names.append(
get_os_package_name(args.rosdistro_name, 'actionlib_msgs'))
print('Always install the following generic dependencies:')
for debian_pkg_name in sorted(debian_pkg_names):
print(' -', debian_pkg_name)
debian_pkg_versions = {}
# get build, run and doc dependencies and map them to binary packages
depends = get_dependencies(pkgs.values(), 'build, run and doc',
_get_build_run_doc_dependencies)
debian_pkg_names_depends = resolve_names(depends, **context)
debian_pkg_names_depends -= set(debian_pkg_names)
debian_pkg_names += order_dependencies(debian_pkg_names_depends)
missing_debian_pkg_names = []
for debian_pkg_name in debian_pkg_names:
try:
debian_pkg_versions.update(
get_binary_package_versions(apt_cache, [debian_pkg_name]))
except KeyError:
missing_debian_pkg_names.append(debian_pkg_name)
if missing_debian_pkg_names:
# we allow missing dependencies to support basic documentation
# of packages which use not released dependencies
print(
'# BEGIN SUBSECTION: MISSING DEPENDENCIES might result in failing build'
)
for debian_pkg_name in missing_debian_pkg_names:
print("Could not find apt package '%s', skipping dependency" %
debian_pkg_name)
debian_pkg_names.remove(debian_pkg_name)
print('# END SUBSECTION')
# generate Dockerfile
data = {
'os_name':
args.os_name,
'os_code_name':
args.os_code_name,
'arch':
args.arch,
'build_tool':
doc_build_file.build_tool,
'distribution_repository_urls':
args.distribution_repository_urls,
'distribution_repository_keys':
get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'environment_variables': [
'ROS_PYTHON_VERSION={}'.format(
condition_context['ROS_PYTHON_VERSION'])
],
'rosdistro_name':
args.rosdistro_name,
'uid':
get_user_id(),
'dependencies':
debian_pkg_names,
'dependency_versions':
debian_pkg_versions,
'install_lists': [],
'canonical_base_url':
doc_build_file.canonical_base_url,
'ordered_pkg_tuples':
ordered_pkg_tuples,
'rosdoc_config_files':
rosdoc_config_files,
}
create_dockerfile('doc/doc_task.Dockerfile.em', data,
args.dockerfile_dir)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the devel job")
parser.add_argument(
'--rosdistro-name',
required=True,
help='The name of the ROS distro to identify the setup file to be '
'sourced')
parser.add_argument('--workspace-root',
nargs='+',
help='The root path of the workspace to compile')
parser.add_argument('--os-name',
required=True,
help="The OS name (e.g. 'ubuntu')")
parser.add_argument('--os-code-name',
required=True,
help="The OS code name (e.g. 'xenial')")
parser.add_argument('--arch',
required=True,
help="The architecture (e.g. 'amd64')")
add_argument_distribution_repository_urls(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_build_tool(parser, required=True)
add_argument_ros_version(parser)
add_argument_env_vars(parser)
add_argument_dockerfile_dir(parser)
add_argument_run_abichecker(parser)
add_argument_require_gpu_support(parser)
a1 = add_argument_build_tool_args(parser)
a2 = add_argument_build_tool_test_args(parser)
parser.add_argument(
'--testing',
action='store_true',
help='The flag if the workspace should be built with tests enabled '
'and instead of installing the tests are ran')
remainder_args = extract_multiple_remainders(argv, (a1, a2))
args = parser.parse_args(argv)
for k, v in remainder_args.items():
setattr(args, k, v)
condition_context = dict(args.env_vars)
condition_context['ROS_DISTRO'] = args.rosdistro_name
condition_context['ROS_VERSION'] = args.ros_version
# get direct build dependencies
pkgs = get_packages_in_workspaces(args.workspace_root, condition_context)
pkg_names = [pkg.name for pkg in pkgs.values()]
print("Found the following packages:")
for pkg_name in sorted(pkg_names):
print(' -', pkg_name)
maintainer_emails = set([])
for pkg in pkgs.values():
for m in pkg.maintainers:
maintainer_emails.add(m.email)
if maintainer_emails:
print('Package maintainer emails: %s' %
' '.join(sorted(maintainer_emails)))
context = initialize_resolver(args.rosdistro_name, args.os_name,
args.os_code_name)
apt_cache = Cache()
debian_pkg_names = [
'build-essential',
'python3',
]
if args.build_tool == 'colcon':
debian_pkg_names += [
'python3-colcon-metadata',
'python3-colcon-output',
'python3-colcon-parallel-executor',
'python3-colcon-ros',
'python3-colcon-test-result',
]
elif 'catkin' not in pkg_names:
debian_pkg_names += resolve_names(['catkin'], **context)
print('Always install the following generic dependencies:')
for debian_pkg_name in sorted(debian_pkg_names):
print(' -', debian_pkg_name)
debian_pkg_versions = {}
# get build dependencies and map them to binary packages
build_depends = get_dependencies(
pkgs.values(), 'build', _get_build_and_recursive_run_dependencies)
debian_pkg_names_building = resolve_names(build_depends, **context)
debian_pkg_names_building -= set(debian_pkg_names)
debian_pkg_names += order_dependencies(debian_pkg_names_building)
debian_pkg_versions.update(
get_binary_package_versions(apt_cache, debian_pkg_names))
# get run and test dependencies and map them to binary packages
run_and_test_depends = get_dependencies(pkgs.values(), 'run and test',
_get_run_and_test_dependencies)
debian_pkg_names_testing = resolve_names(run_and_test_depends, **context)
# all additional run/test dependencies
# are added after the build dependencies
# in order to reuse existing images in the docker container
debian_pkg_names_testing -= set(debian_pkg_names)
debian_pkg_versions.update(
get_binary_package_versions(apt_cache, debian_pkg_names_testing))
if args.testing:
debian_pkg_names += order_dependencies(debian_pkg_names_testing)
mapped_workspaces = [
(workspace_root, '/tmp/ws%s' % (index if index > 1 else ''))
for index, workspace_root in enumerate(args.workspace_root, 1)
]
parent_result_space = []
if len(args.workspace_root) > 1:
parent_result_space = ['/opt/ros/%s' % args.rosdistro_name] + \
[mapping[1] for mapping in mapped_workspaces[:-1]]
# generate Dockerfile
data = {
'os_name':
args.os_name,
'os_code_name':
args.os_code_name,
'arch':
args.arch,
'distribution_repository_urls':
args.distribution_repository_urls,
'distribution_repository_keys':
get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'rosdistro_name':
args.rosdistro_name,
'uid':
get_user_id(),
'build_tool':
args.build_tool,
'build_tool_args':
args.build_tool_args,
'build_tool_test_args':
args.build_tool_test_args,
'ros_version':
args.ros_version,
'build_environment_variables':
['%s=%s' % key_value for key_value in args.env_vars.items()],
'dependencies':
debian_pkg_names,
'dependency_versions':
debian_pkg_versions,
'install_lists': [],
'testing':
args.testing,
'run_abichecker':
args.run_abichecker,
'require_gpu_support':
args.require_gpu_support,
'workspace_root':
mapped_workspaces[-1][1],
'parent_result_space':
parent_result_space,
}
create_dockerfile('devel/devel_task.Dockerfile.em', data,
args.dockerfile_dir)
# output hints about necessary volumes to mount
ros_buildfarm_basepath = os.path.normpath(
os.path.join(os.path.dirname(__file__), '..', '..'))
print('Mount the following volumes when running the container:')
print(' -v %s:/tmp/ros_buildfarm:ro' % ros_buildfarm_basepath)
for mapping in mapped_workspaces:
print(' -v %s:%s' % mapping)
def main(argv=sys.argv[1:]):
parser = argparse.ArgumentParser(
description="Generate a 'Dockerfile' for the devel job")
parser.add_argument(
'--rosdistro-name',
required=True,
help='The name of the ROS distro to identify the setup file to be '
'sourced')
parser.add_argument('--workspace-root',
nargs='+',
help='The root path of the workspace to compile')
parser.add_argument('--os-name',
required=True,
help="The OS name (e.g. 'ubuntu')")
parser.add_argument('--os-code-name',
required=True,
help="The OS code name (e.g. 'trusty')")
parser.add_argument('--arch',
required=True,
help="The architecture (e.g. 'amd64')")
add_argument_distribution_repository_urls(parser)
add_argument_distribution_repository_key_files(parser)
add_argument_dockerfile_dir(parser)
parser.add_argument(
'--testing',
action='store_true',
help='The flag if the workspace should be built with tests enabled '
'and instead of installing the tests are ran')
args = parser.parse_args(argv)
# get direct build dependencies
pkgs = {}
for workspace_root in args.workspace_root:
source_space = os.path.join(workspace_root, 'src')
print("Crawling for packages in workspace '%s'" % source_space)
pkgs.update(find_packages(source_space))
pkg_names = [pkg.name for pkg in pkgs.values()]
print("Found the following packages:")
for pkg_name in sorted(pkg_names):
print(' -', pkg_name)
maintainer_emails = set([])
for pkg in pkgs.values():
for m in pkg.maintainers:
maintainer_emails.add(m.email)
if maintainer_emails:
print('Package maintainer emails: %s' %
' '.join(sorted(maintainer_emails)))
context = initialize_resolver(args.rosdistro_name, args.os_name,
args.os_code_name)
apt_cache = Cache()
debian_pkg_names = [
'build-essential',
'python3',
]
if 'catkin' not in pkg_names:
debian_pkg_names.append(
get_debian_package_name(args.rosdistro_name, 'catkin'))
print('Always install the following generic dependencies:')
for debian_pkg_name in sorted(debian_pkg_names):
print(' -', debian_pkg_name)
debian_pkg_versions = {}
# get build dependencies and map them to binary packages
build_depends = get_dependencies(
pkgs.values(), 'build', _get_build_and_recursive_run_dependencies)
debian_pkg_names_building = resolve_names(build_depends, **context)
debian_pkg_names_building -= set(debian_pkg_names)
debian_pkg_names += order_dependencies(debian_pkg_names_building)
debian_pkg_versions.update(
get_binary_package_versions(apt_cache, debian_pkg_names))
# get run and test dependencies and map them to binary packages
run_and_test_depends = get_dependencies(pkgs.values(), 'run and test',
_get_run_and_test_dependencies)
debian_pkg_names_testing = resolve_names(run_and_test_depends, **context)
# all additional run/test dependencies
# are added after the build dependencies
# in order to reuse existing images in the docker container
debian_pkg_names_testing -= set(debian_pkg_names)
debian_pkg_versions.update(
get_binary_package_versions(apt_cache, debian_pkg_names_testing))
if args.testing:
debian_pkg_names += order_dependencies(debian_pkg_names_testing)
# generate Dockerfile
data = {
'os_name':
args.os_name,
'os_code_name':
args.os_code_name,
'arch':
args.arch,
'distribution_repository_urls':
args.distribution_repository_urls,
'distribution_repository_keys':
get_distribution_repository_keys(
args.distribution_repository_urls,
args.distribution_repository_key_files),
'rosdistro_name':
args.rosdistro_name,
'uid':
get_user_id(),
'dependencies':
debian_pkg_names,
'dependency_versions':
debian_pkg_versions,
'testing':
args.testing,
'prerelease_overlay':
len(args.workspace_root) > 1,
}
create_dockerfile('devel/devel_task.Dockerfile.em', data,
args.dockerfile_dir)
# output hints about necessary volumes to mount
ros_buildfarm_basepath = os.path.normpath(
os.path.join(os.path.dirname(__file__), '..', '..'))
print('Mount the following volumes when running the container:')
print(' -v %s:/tmp/ros_buildfarm:ro' % ros_buildfarm_basepath)
print(' -v %s:/tmp/catkin_workspace' % args.workspace_root[-1])