def _build_dependencies(self): package_files = self.download_dependencies() def filter_files(package_files): for p in package_files: banned = False for k in self.BANNED_PACKAGES: if k in p.lower(): banned = True if banned: continue yield p package_files = [f for f in filter_files(package_files)] if not package_files: LOG.info("No RPM packages of OpenStack dependencies to build") return package_base_names = [sh.basename(f) for f in package_files] utils.log_iterable(sorted(package_base_names), logger=LOG, header=("Building %s dependency RPM" " packages") % (len(package_files))) with utils.progress_bar(name='Building', max_am=len(package_files)) as p_bar: for (i, filename) in enumerate(sorted(package_files)): cmdline = self.py2rpm_start_cmdline() + ["--", filename] build_filename = "py2rpm-%s.out" % sh.basename(filename) out_filename = sh.joinpths(self.log_dir, build_filename) sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True) p_bar.update(i + 1)
def _uninstall_pkgs(self): pkgs = self.tracereader.packages_installed() if pkgs: pkg_names = set([p['name'] for p in pkgs]) utils.log_iterable( pkg_names, logger=LOG, header="Potentially removing %s distribution packages" % (len(pkg_names))) which_removed = [] with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: for (i, p) in enumerate(pkgs): uninstaller = make_packager( p, self.distro.package_manager_class, distro=self.distro, remove_default=self.purge_packages) if uninstaller.remove(p): which_removed.append(p['name']) p_bar.update(i + 1) utils.log_iterable( which_removed, logger=LOG, header="Actually removed %s distribution packages" % (len(which_removed)))
def _uninstall_pips(self): pips = self.tracereader.pips_installed() if pips: pip_names = [p["name"] for p in pips] utils.log_iterable( pip_names, logger=LOG, header="Potentially removing %s python packages" % (len(pip_names)) ) which_removed = [] with utils.progress_bar("Uninstalling", len(pips), reverse=True) as p_bar: for (i, p) in enumerate(pips): try: uninstaller = make_packager( p, pip.Packager, distro=self.distro, remove_default=self.purge_packages ) if uninstaller.remove(p): which_removed.append(p["name"]) except excp.ProcessExecutionError as e: # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode combined = str(e.stderr) + str(e.stdout) if not re.search(r"not\s+installed", combined, re.I): raise p_bar.update(i + 1) utils.log_iterable( which_removed, logger=LOG, header="Actually removed %s python packages" % (len(which_removed)) )
def _uninstall_pips(self): pips = self.tracereader.pips_installed() if pips: pip_names = [p['name'] for p in pips] utils.log_iterable( pip_names, logger=LOG, header="Potentially removing %s python packages" % (len(pip_names))) which_removed = [] with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar: for (i, p) in enumerate(pips): try: uninstaller = make_packager( p, pip.Packager, distro=self.distro, remove_default=self.purge_packages) if uninstaller.remove(p): which_removed.append(p['name']) except excp.ProcessExecutionError as e: # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode combined = (str(e.stderr) + str(e.stdout)) if not re.search(r"not\s+installed", combined, re.I): raise p_bar.update(i + 1) utils.log_iterable(which_removed, logger=LOG, header="Actually removed %s python packages" % (len(which_removed)))
def _get_log(self): log_cmd = ['git', 'log', '--pretty=oneline', '--color=never'] (sysout, _stderr) = sh.execute(*log_cmd, cwd=self.wkdir) lines = sysout.strip('\n').splitlines() # Extract the raw commit details try: mmp = self._parse_mailmap() except IOError: mmp = {} log = [] with utils.progress_bar("Git changelog analysis", len(lines) + 1) as pb: for i in range(0, len(lines), PER_CALL_AM): pb.update(i) line = lines[i] fields = line.split(' ') if not len(fields): continue # See: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt commit_id = fields[0] details = self._get_commit_detail(commit_id, "[%s][%ai][%aE][%an]", PER_CALL_AM) for det in details.splitlines(): details_m = re.match( r"^\s*\[(.*?)\]\[(.*?)\]\[(.*?)\]\[(.*?)\]\s*$", det) if not details_m: continue (summary, date, author_email, author_name) = details_m.groups() author_email = mmp.get(author_email, author_email) date = iso8601.parse_date(date) if self._skip_entry(summary, date, author_email, author_name): continue log.append({ 'summary': translate_utf8(summary), 'when': date, 'author_email': translate_utf8(author_email), 'author_name': translate_utf8(author_name), }) # Bucketize the dates by day date_buckets = {} for entry in log: day = entry['when'].date() if day in date_buckets: date_buckets[day].append(entry) else: date_buckets[day] = [entry] return date_buckets
def _install_pips(self): pips = self._get_pips() if pips: pip_names = [p['name'] for p in pips] utils.log_iterable(pip_names, logger=LOG, header="Setting up %s python packages" % (len(pip_names))) with utils.progress_bar('Installing', len(pips)) as p_bar: for (i, p) in enumerate(pips): self.tracewriter.pip_installed(p) self.pip_factory.get_packager_for(p).install(p) p_bar.update(i + 1)
def _install_pips(self): pips = self.pips if pips: pip_names = [p['name'] for p in pips] utils.log_iterable(pip_names, logger=LOG, header="Setting up %s python packages" % (len(pip_names))) with utils.progress_bar('Installing', len(pips)) as p_bar: for (i, p) in enumerate(pips): self.tracewriter.pip_installed(p) installer = make_packager(p, pip.Packager, distro=self.distro) installer.install(p) p_bar.update(i + 1)
def _install_pips(self): pips = self.pips if pips: pip_names = [p["name"] for p in pips] utils.log_iterable(pip_names, logger=LOG, header="Setting up %s python packages" % (len(pip_names))) with utils.progress_bar("Installing", len(pips)) as p_bar: for (i, p) in enumerate(pips): installer = make_packager(p, pip.Packager, distro=self.distro) installer.install(p) # Note that we did it so that we can remove it... self.tracewriter.pip_installed(filter_package(p)) p_bar.update(i + 1)
def install(self): LOG.debug("Preparing to install packages for: %r", self.name) pkgs = self.packages if pkgs: pkg_names = [p["name"] for p in pkgs] utils.log_iterable(pkg_names, logger=LOG, header="Setting up %s distribution packages" % (len(pkg_names))) with utils.progress_bar("Installing", len(pkgs)) as p_bar: for (i, p) in enumerate(pkgs): installer = make_packager(p, self.distro.package_manager_class, distro=self.distro) installer.install(p) # Mark that this happened so that we can uninstall it self.tracewriter.package_installed(filter_package(p)) p_bar.update(i + 1)
def install(self): LOG.debug('Preparing to install packages for: %r', self.name) pkgs = self._get_packages() if pkgs: pkg_names = [p['name'] for p in pkgs] utils.log_iterable(pkg_names, logger=LOG, header="Setting up %s distribution packages" % (len(pkg_names))) with utils.progress_bar('Installing', len(pkgs)) as p_bar: for (i, p) in enumerate(pkgs): self.tracewriter.package_installed(p) self.packager_factory.get_packager_for(p).install(p) p_bar.update(i + 1) return self.get_option('trace_dir')
def install(self): LOG.debug('Preparing to install packages for: %r', self.name) pkgs = self.packages if pkgs: pkg_names = [p['name'] for p in pkgs] utils.log_iterable(pkg_names, logger=LOG, header="Setting up %s distribution packages" % (len(pkg_names))) with utils.progress_bar('Installing', len(pkgs)) as p_bar: for (i, p) in enumerate(pkgs): installer = make_packager(p, self.distro.package_manager_class, distro=self.distro) self.tracewriter.package_installed(p) installer.install(p) p_bar.update(i + 1)
def _get_log(self): log_cmd = ['git', 'log', '--pretty=oneline', '--color=never'] if self.max_history > 0: log_cmd += ['-n%s' % (self.max_history)] (sysout, _stderr) = sh.execute(*log_cmd, cwd=self.wkdir) lines = sysout.strip('\n').splitlines() # Extract the raw commit details try: mmp = self._parse_mailmap() except IOError: mmp = {} log = [] with utils.progress_bar("Git changelog analysis", len(lines) + 1) as pb: for i in range(0, len(lines), PER_CALL_AM): pb.update(i) line = lines[i] fields = line.split(' ') if not len(fields): continue # See: http://opensource.apple.com/source/Git/Git-26/src/git-htmldocs/pretty-formats.txt commit_id = fields[0] details = self._get_commit_detail(commit_id, "[%s][%ai][%aE][%an]", PER_CALL_AM) for det in details.splitlines(): details_m = re.match(r"^\s*\[(.*?)\]\[(.*?)\]\[(.*?)\]\[(.*?)\]\s*$", det) if not details_m: continue (summary, date, author_email, author_name) = details_m.groups() author_email = mmp.get(author_email, author_email) date = iso8601.parse_date(date) if self._skip_entry(summary, date, author_email, author_name): continue log.append({ 'summary': summary, 'when': date, 'author_email': author_email, 'author_name': author_name, }) # Bucketize the dates by day date_buckets = {} for entry in log: day = entry['when'].date() if day in date_buckets: date_buckets[day].append(entry) else: date_buckets[day] = [entry] return date_buckets
def _install_pips(self): pips = self.pips if pips: pip_names = [p['name'] for p in pips] utils.log_iterable(pip_names, logger=LOG, header="Setting up %s python packages" % (len(pip_names))) with utils.progress_bar('Installing', len(pips)) as p_bar: for (i, p) in enumerate(pips): installer = make_packager(p, pip.Packager, distro=self.distro) installer.install(p) # Note that we did it so that we can remove it... self.tracewriter.pip_installed(filter_package(p)) p_bar.update(i + 1)
def install(self): LOG.debug('Preparing to install packages for: %r', self.name) pkgs = self.packages if pkgs: pkg_names = [p['name'] for p in pkgs] utils.log_iterable(pkg_names, logger=LOG, header="Setting up %s distribution packages" % (len(pkg_names))) with utils.progress_bar('Installing', len(pkgs)) as p_bar: for (i, p) in enumerate(pkgs): installer = make_packager(p, self.distro.package_manager_class, distro=self.distro) installer.install(p) # Mark that this happened so that we can uninstall it self.tracewriter.package_installed(filter_package(p)) p_bar.update(i + 1)
def _uninstall_pkgs(self): pkgs = self.tracereader.packages_installed() if pkgs: pkg_names = set([p['name'] for p in pkgs]) utils.log_iterable(pkg_names, logger=LOG, header="Potentially removing %s distribution packages" % (len(pkg_names))) which_removed = [] with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: for (i, p) in enumerate(pkgs): uninstaller = make_packager(p, self.distro.package_manager_class, distro=self.distro, remove_default=self.purge_packages) if uninstaller.remove(p): which_removed.append(p['name']) p_bar.update(i + 1) utils.log_iterable(which_removed, logger=LOG, header="Actually removed %s distribution packages" % (len(which_removed)))
def _uninstall_pkgs(self): if self.get_option('keep_old', False): LOG.info('Keep-old flag set, not removing any packages.') else: pkgs = self.tracereader.packages_installed() if pkgs: pkg_names = set([p['name'] for p in pkgs]) utils.log_iterable(pkg_names, logger=LOG, header="Potentially removing %s packages" % (len(pkg_names))) which_removed = set() with utils.progress_bar('Uninstalling', len(pkgs), reverse=True) as p_bar: for (i, p) in enumerate(pkgs): if self.packager_factory.get_packager_for(p).remove(p): which_removed.add(p['name']) p_bar.update(i + 1) utils.log_iterable(which_removed, logger=LOG, header="Actually removed %s packages" % (len(which_removed)))
def _build_openstack(self): if not self.package_dirs: LOG.warn("No RPM packages of OpenStack installs to build") return component_names = [self._get_component_name(d) for d in self.package_dirs] utils.log_iterable(sorted(component_names), logger=LOG, header=("Building %s OpenStack RPM" " packages") % (len(self.package_dirs))) with utils.progress_bar(name='Building', max_am=len(self.package_dirs)) as p_bar: for (i, pkg_dir) in enumerate(sorted(self.package_dirs)): component_name = self._get_component_name(pkg_dir) cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir] out_filename = sh.joinpths(self.log_dir, "py2rpm.%s.out" % (component_name)) sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True) p_bar.update(i + 1)
def _uninstall_pips(self): if self.get_option('keep_old', False): LOG.info('Keep-old flag set, not removing any python packages.') else: pips = self.tracereader.pips_installed() if pips: pip_names = set([p['name'] for p in pips]) utils.log_iterable(pip_names, logger=LOG, header="Uninstalling %s python packages" % (len(pip_names))) with utils.progress_bar('Uninstalling', len(pips), reverse=True) as p_bar: for (i, p) in enumerate(pips): try: self.pip_factory.get_packager_for(p).remove(p) except excp.ProcessExecutionError as e: # NOTE(harlowja): pip seems to die if a pkg isn't there even in quiet mode combined = (str(e.stderr) + str(e.stdout)) if not re.search(r"not\s+installed", combined, re.I): raise p_bar.update(i + 1)
def _build_openstack(self): if not self.package_dirs: LOG.warn("No RPM packages of OpenStack installs to build") return component_names = [ self._get_component_name(d) for d in self.package_dirs ] utils.log_iterable(sorted(component_names), logger=LOG, header=("Building %s OpenStack RPM" " packages") % (len(self.package_dirs))) with utils.progress_bar(name='Building', max_am=len(self.package_dirs)) as p_bar: for (i, pkg_dir) in enumerate(sorted(self.package_dirs)): component_name = self._get_component_name(pkg_dir) cmdline = self.py2rpm_start_cmdline() + ["--", pkg_dir] out_filename = sh.joinpths(self.log_dir, "py2rpm.%s.out" % (component_name)) sh.execute_save_output(cmdline, out_filename=out_filename, quiet=True) p_bar.update(i + 1)