def _build_dependencies(self):
package_files = self.download_dependencies()
def filter_files(package_files):
for p in package_files:
banned = False
for k in self.BANNED_PACKAGES:
if k in p.lower():
banned = True
if banned:
continue
yield p
package_files = [f for f in filter_files(package_files)]
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
return
package_base_names = [sh.basename(f) for f in package_files]
utils.log_iterable(sorted(package_base_names), logger=LOG,
header=("Building %s dependency RPM"
" packages") % (len(package_files)))
with utils.progress_bar(name='Building',
max_am=len(package_files)) as p_bar:
for (i, filename) in enumerate(sorted(package_files)):
cmdline = self.py2rpm_start_cmdline() + ["--", filename]
build_filename = "py2rpm-%s.out" % sh.basename(filename)
out_filename = sh.joinpths(self.log_dir, build_filename)
sh.execute_save_output(cmdline, out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _uninstall_pkgs(self):
pkgs = self.tracereader.packages_installed()
if pkgs:
pkg_names = set([p['name'] for p in pkgs])
utils.log_iterable(
pkg_names,
logger=LOG,
header="Potentially removing %s distribution packages" %
(len(pkg_names)))
which_removed = []
with utils.progress_bar('Uninstalling', len(pkgs),
reverse=True) as p_bar:
for (i, p) in enumerate(pkgs):
uninstaller = make_packager(
p,
self.distro.package_manager_class,
distro=self.distro,
remove_default=self.purge_packages)
if uninstaller.remove(p):
which_removed.append(p['name'])
p_bar.update(i + 1)
utils.log_iterable(
which_removed,
logger=LOG,
header="Actually removed %s distribution packages" %
(len(which_removed)))
def _uninstall_pips(self):
pips = self.tracereader.pips_installed()
if pips:
pip_names = [p["name"] for p in pips]
utils.log_iterable(
pip_names, logger=LOG, header="Potentially removing %s python packages" % (len(pip_names))
)
which_removed = []
with utils.progress_bar("Uninstalling", len(pips), reverse=True) as p_bar:
for (i, p) in enumerate(pips):
try:
uninstaller = make_packager(
p, pip.Packager, distro=self.distro, remove_default=self.purge_packages
)
if uninstaller.remove(p):
which_removed.append(p["name"])
except excp.ProcessExecutionError as e:
# NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode
combined = str(e.stderr) + str(e.stdout)
if not re.search(r"not\s+installed", combined, re.I):
raise
p_bar.update(i + 1)
utils.log_iterable(
which_removed, logger=LOG, header="Actually removed %s python packages" % (len(which_removed))
)
def _uninstall_pips(self):
pips = self.tracereader.pips_installed()
if pips:
pip_names = [p['name'] for p in pips]
utils.log_iterable(
pip_names,
logger=LOG,
header="Potentially removing %s python packages" %
(len(pip_names)))
which_removed = []
with utils.progress_bar('Uninstalling', len(pips),
reverse=True) as p_bar:
for (i, p) in enumerate(pips):
try:
uninstaller = make_packager(
p,
pip.Packager,
distro=self.distro,
remove_default=self.purge_packages)
if uninstaller.remove(p):
which_removed.append(p['name'])
except excp.ProcessExecutionError as e:
# NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode
combined = (str(e.stderr) + str(e.stdout))
if not re.search(r"not\s+installed", combined, re.I):
raise
p_bar.update(i + 1)
utils.log_iterable(which_removed,
logger=LOG,
header="Actually removed %s python packages" %
(len(which_removed)))
def _build_dependencies(self):
package_files = self.download_dependencies()
def filter_files(package_files):
for p in package_files:
banned = False
for k in self.BANNED_PACKAGES:
if k in p.lower():
banned = True
if banned:
continue
yield p
package_files = [f for f in filter_files(package_files)]
if not package_files:
LOG.info("No RPM packages of OpenStack dependencies to build")
return
package_base_names = [sh.basename(f) for f in package_files]
utils.log_iterable(sorted(package_base_names),
logger=LOG,
header=("Building %s dependency RPM"
" packages") % (len(package_files)))
with utils.progress_bar(name='Building',
max_am=len(package_files)) as p_bar:
for (i, filename) in enumerate(sorted(package_files)):
cmdline = self.py2rpm_start_cmdline() + ["--", filename]
build_filename = "py2rpm-%s.out" % sh.basename(filename)
out_filename = sh.joinpths(self.log_dir, build_filename)
sh.execute_save_output(cmdline,
out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _get_log(self):
log_cmd = ['git', 'log', '--pretty=oneline', '--color=never']
(sysout, _stderr) = sh.execute(*log_cmd, cwd=self.wkdir)
lines = sysout.strip('\n').splitlines()
# Extract the raw commit details
try:
mmp = self._parse_mailmap()
except IOError:
mmp = {}
log = []
with utils.progress_bar("Git changelog analysis",
len(lines) + 1) as pb:
for i in range(0, len(lines), PER_CALL_AM):
pb.update(i)
line = lines[i]
fields = line.split(' ')
if not len(fields):
continue
# See: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt
commit_id = fields[0]
details = self._get_commit_detail(commit_id,
"[%s][%ai][%aE][%an]",
PER_CALL_AM)
for det in details.splitlines():
details_m = re.match(
r"^\s*\[(.*?)\]\[(.*?)\]\[(.*?)\]\[(.*?)\]\s*$", det)
if not details_m:
continue
(summary, date, author_email,
author_name) = details_m.groups()
author_email = mmp.get(author_email, author_email)
date = iso8601.parse_date(date)
if self._skip_entry(summary, date, author_email,
author_name):
continue
log.append({
'summary': translate_utf8(summary),
'when': date,
'author_email': translate_utf8(author_email),
'author_name': translate_utf8(author_name),
})
# Bucketize the dates by day
date_buckets = {}
for entry in log:
day = entry['when'].date()
if day in date_buckets:
date_buckets[day].append(entry)
else:
date_buckets[day] = [entry]
return date_buckets
def _install_pips(self):
pips = self._get_pips()
if pips:
pip_names = [p['name'] for p in pips]
utils.log_iterable(pip_names, logger=LOG,
header="Setting up %s python packages" % (len(pip_names)))
with utils.progress_bar('Installing', len(pips)) as p_bar:
for (i, p) in enumerate(pips):
self.tracewriter.pip_installed(p)
self.pip_factory.get_packager_for(p).install(p)
p_bar.update(i + 1)
def _install_pips(self):
pips = self.pips
if pips:
pip_names = [p['name'] for p in pips]
utils.log_iterable(pip_names, logger=LOG,
header="Setting up %s python packages" % (len(pip_names)))
with utils.progress_bar('Installing', len(pips)) as p_bar:
for (i, p) in enumerate(pips):
self.tracewriter.pip_installed(p)
installer = make_packager(p, pip.Packager, distro=self.distro)
installer.install(p)
p_bar.update(i + 1)
def _install_pips(self):
pips = self.pips
if pips:
pip_names = [p["name"] for p in pips]
utils.log_iterable(pip_names, logger=LOG, header="Setting up %s python packages" % (len(pip_names)))
with utils.progress_bar("Installing", len(pips)) as p_bar:
for (i, p) in enumerate(pips):
installer = make_packager(p, pip.Packager, distro=self.distro)
installer.install(p)
# Note that we did it so that we can remove it...
self.tracewriter.pip_installed(filter_package(p))
p_bar.update(i + 1)
def install(self):
LOG.debug("Preparing to install packages for: %r", self.name)
pkgs = self.packages
if pkgs:
pkg_names = [p["name"] for p in pkgs]
utils.log_iterable(pkg_names, logger=LOG, header="Setting up %s distribution packages" % (len(pkg_names)))
with utils.progress_bar("Installing", len(pkgs)) as p_bar:
for (i, p) in enumerate(pkgs):
installer = make_packager(p, self.distro.package_manager_class, distro=self.distro)
installer.install(p)
# Mark that this happened so that we can uninstall it
self.tracewriter.package_installed(filter_package(p))
p_bar.update(i + 1)
def install(self):
LOG.debug('Preparing to install packages for: %r', self.name)
pkgs = self._get_packages()
if pkgs:
pkg_names = [p['name'] for p in pkgs]
utils.log_iterable(pkg_names, logger=LOG,
header="Setting up %s distribution packages" % (len(pkg_names)))
with utils.progress_bar('Installing', len(pkgs)) as p_bar:
for (i, p) in enumerate(pkgs):
self.tracewriter.package_installed(p)
self.packager_factory.get_packager_for(p).install(p)
p_bar.update(i + 1)
return self.get_option('trace_dir')
def install(self):
LOG.debug('Preparing to install packages for: %r', self.name)
pkgs = self.packages
if pkgs:
pkg_names = [p['name'] for p in pkgs]
utils.log_iterable(pkg_names, logger=LOG,
header="Setting up %s distribution packages" % (len(pkg_names)))
with utils.progress_bar('Installing', len(pkgs)) as p_bar:
for (i, p) in enumerate(pkgs):
installer = make_packager(p, self.distro.package_manager_class, distro=self.distro)
self.tracewriter.package_installed(p)
installer.install(p)
p_bar.update(i + 1)
def _get_log(self):
log_cmd = ['git', 'log', '--pretty=oneline', '--color=never']
if self.max_history > 0:
log_cmd += ['-n%s' % (self.max_history)]
(sysout, _stderr) = sh.execute(*log_cmd, cwd=self.wkdir)
lines = sysout.strip('\n').splitlines()
# Extract the raw commit details
try:
mmp = self._parse_mailmap()
except IOError:
mmp = {}
log = []
with utils.progress_bar("Git changelog analysis", len(lines) + 1) as pb:
for i in range(0, len(lines), PER_CALL_AM):
pb.update(i)
line = lines[i]
fields = line.split(' ')
if not len(fields):
continue
# See: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt
commit_id = fields[0]
details = self._get_commit_detail(commit_id, "[%s][%ai][%aE][%an]", PER_CALL_AM)
for det in details.splitlines():
details_m = re.match(r"^\s*\[(.*?)\]\[(.*?)\]\[(.*?)\]\[(.*?)\]\s*$", det)
if not details_m:
continue
(summary, date, author_email, author_name) = details_m.groups()
author_email = mmp.get(author_email, author_email)
date = iso8601.parse_date(date)
if self._skip_entry(summary, date, author_email, author_name):
continue
log.append({
'summary': summary,
'when': date,
'author_email': author_email,
'author_name': author_name,
})
# Bucketize the dates by day
date_buckets = {}
for entry in log:
day = entry['when'].date()
if day in date_buckets:
date_buckets[day].append(entry)
else:
date_buckets[day] = [entry]
return date_buckets
def _install_pips(self):
pips = self.pips
if pips:
pip_names = [p['name'] for p in pips]
utils.log_iterable(pip_names, logger=LOG,
header="Setting up %s python packages" % (len(pip_names)))
with utils.progress_bar('Installing', len(pips)) as p_bar:
for (i, p) in enumerate(pips):
installer = make_packager(p, pip.Packager,
distro=self.distro)
installer.install(p)
# Note that we did it so that we can remove it...
self.tracewriter.pip_installed(filter_package(p))
p_bar.update(i + 1)
def install(self):
LOG.debug('Preparing to install packages for: %r', self.name)
pkgs = self.packages
if pkgs:
pkg_names = [p['name'] for p in pkgs]
utils.log_iterable(pkg_names, logger=LOG,
header="Setting up %s distribution packages" % (len(pkg_names)))
with utils.progress_bar('Installing', len(pkgs)) as p_bar:
for (i, p) in enumerate(pkgs):
installer = make_packager(p, self.distro.package_manager_class,
distro=self.distro)
installer.install(p)
# Mark that this happened so that we can uninstall it
self.tracewriter.package_installed(filter_package(p))
p_bar.update(i + 1)
def _uninstall_pkgs(self):
pkgs = self.tracereader.packages_installed()
if pkgs:
pkg_names = set([p['name'] for p in pkgs])
utils.log_iterable(pkg_names, logger=LOG,
header="Potentially removing %s distribution packages" % (len(pkg_names)))
which_removed = []
with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar:
for (i, p) in enumerate(pkgs):
uninstaller = make_packager(p, self.distro.package_manager_class,
distro=self.distro, remove_default=self.purge_packages)
if uninstaller.remove(p):
which_removed.append(p['name'])
p_bar.update(i + 1)
utils.log_iterable(which_removed, logger=LOG,
header="Actually removed %s distribution packages" % (len(which_removed)))
def _uninstall_pkgs(self):
if self.get_option('keep_old', False):
LOG.info('Keep-old flag set, not removing any packages.')
else:
pkgs = self.tracereader.packages_installed()
if pkgs:
pkg_names = set([p['name'] for p in pkgs])
utils.log_iterable(pkg_names, logger=LOG,
header="Potentially removing %s packages" % (len(pkg_names)))
which_removed = set()
with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar:
for (i, p) in enumerate(pkgs):
if self.packager_factory.get_packager_for(p).remove(p):
which_removed.add(p['name'])
p_bar.update(i + 1)
utils.log_iterable(which_removed, logger=LOG,
header="Actually removed %s packages" % (len(which_removed)))
def _build_openstack(self):
if not self.package_dirs:
LOG.warn("No RPM packages of OpenStack installs to build")
return
component_names = [self._get_component_name(d)
for d in self.package_dirs]
utils.log_iterable(sorted(component_names), logger=LOG,
header=("Building %s OpenStack RPM"
" packages") % (len(self.package_dirs)))
with utils.progress_bar(name='Building',
max_am=len(self.package_dirs)) as p_bar:
for (i, pkg_dir) in enumerate(sorted(self.package_dirs)):
component_name = self._get_component_name(pkg_dir)
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
out_filename = sh.joinpths(self.log_dir,
"py2rpm.%s.out" % (component_name))
sh.execute_save_output(cmdline, out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
def _uninstall_pips(self):
if self.get_option('keep_old', False):
LOG.info('Keep-old flag set, not removing any python packages.')
else:
pips = self.tracereader.pips_installed()
if pips:
pip_names = set([p['name'] for p in pips])
utils.log_iterable(pip_names, logger=LOG,
header="Uninstalling %s python packages" % (len(pip_names)))
with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar:
for (i, p) in enumerate(pips):
try:
self.pip_factory.get_packager_for(p).remove(p)
except excp.ProcessExecutionError as e:
# NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode
combined = (str(e.stderr) + str(e.stdout))
if not re.search(r"not\s+installed", combined, re.I):
raise
p_bar.update(i + 1)
def _build_openstack(self):
if not self.package_dirs:
LOG.warn("No RPM packages of OpenStack installs to build")
return
component_names = [
self._get_component_name(d) for d in self.package_dirs
]
utils.log_iterable(sorted(component_names),
logger=LOG,
header=("Building %s OpenStack RPM"
" packages") % (len(self.package_dirs)))
with utils.progress_bar(name='Building',
max_am=len(self.package_dirs)) as p_bar:
for (i, pkg_dir) in enumerate(sorted(self.package_dirs)):
component_name = self._get_component_name(pkg_dir)
cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir]
out_filename = sh.joinpths(self.log_dir,
"py2rpm.%s.out" % (component_name))
sh.execute_save_output(cmdline,
out_filename=out_filename,
quiet=True)
p_bar.update(i + 1)
