def sign_manifest(self, myupdates, myremoved, mymanifests): try: for x in sorted(vcs_files_to_cps( chain(myupdates, myremoved, mymanifests), self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)): self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) manifest_path = os.path.join(self.repoman_settings["O"], "Manifest") if not need_signature(manifest_path): continue gpgsign(manifest_path, self.repoman_settings, self.options) except portage.exception.PortageException as e: portage.writemsg("!!! %s\n" % str(e)) portage.writemsg("!!! Disabled FEATURES='sign'\n") self.repo_settings.sign_manifests = False
def sign_manifest(self, myupdates, myremoved, mymanifests): try: for x in sorted(vcs_files_to_cps( chain(myupdates, myremoved, mymanifests), self.scanner.repo_settings.repodir, self.scanner.repolevel, self.scanner.reposplit, self.scanner.categories)): self.repoman_settings["O"] = os.path.join(self.repo_settings.repodir, x) manifest_path = os.path.join(self.repoman_settings["O"], "Manifest") if not need_signature(manifest_path): continue gpgsign(manifest_path, self.repoman_settings, self.options) except portage.exception.PortageException as e: portage.writemsg("!!! %s\n" % str(e)) portage.writemsg("!!! Disabled FEATURES='sign'\n") self.repo_settings.sign_manifests = False
async def _async_test_simple( self, playground, metadata_xml_files, profiles, profile, licenses, arch_list, use_desc, metadata_xsd, copyright_header, debug, ): settings = playground.settings eprefix = settings["EPREFIX"] eroot = settings["EROOT"] portdb = playground.trees[playground.eroot]["porttree"].dbapi homedir = os.path.join(eroot, "home") distdir = os.path.join(eprefix, "distdir") test_repo_location = settings.repositories["test_repo"].location profiles_dir = os.path.join(test_repo_location, "profiles") license_dir = os.path.join(test_repo_location, "licenses") repoman_cmd = (portage._python_interpreter, "-b", "-Wd", os.path.join(self.bindir, "repoman")) git_binary = find_binary("git") git_cmd = (git_binary, ) cp_binary = find_binary("cp") self.assertEqual(cp_binary is None, False, "cp command not found") cp_cmd = (cp_binary, ) test_ebuild = portdb.findname("dev-libs/A-1") self.assertFalse(test_ebuild is None) committer_name = "Gentoo Dev" committer_email = "*****@*****.**" expected_warnings = { "returncode": 0, "warns": { "variable.phase": [ "dev-libs/C/C-0.ebuild: line 15: phase pkg_preinst: EAPI 7: variable A: Forbidden reference to variable specified by PMS", "dev-libs/C/C-0.ebuild: line 15: phase pkg_preinst: EAPI 7: variable BROOT: Forbidden reference to variable specified by PMS", ] }, } git_test = ( ("", RepomanRun(args=["--version"])), ("", RepomanRun(args=["manifest"])), ("", git_cmd + ( "config", "--global", "user.name", committer_name, )), ("", git_cmd + ( "config", "--global", "user.email", committer_email, )), ("", git_cmd + ("init-db", )), ("", git_cmd + ("add", ".")), ("", git_cmd + ("commit", "-a", "-m", "add whole repo")), ("", RepomanRun(args=["full", "-d"], expected=expected_warnings)), ("", RepomanRun(args=[ "full", "--include-profiles", "default/linux/x86/test_profile" ], expected=expected_warnings)), ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "2.ebuild")), ("", git_cmd + ("add", test_ebuild[:-8] + "2.ebuild")), ("", RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 2"], expected=expected_warnings)), ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "3.ebuild")), ("", git_cmd + ("add", test_ebuild[:-8] + "3.ebuild")), ("dev-libs", RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 3"], expected=expected_warnings)), ("", cp_cmd + (test_ebuild, test_ebuild[:-8] + "4.ebuild")), ("", git_cmd + ("add", test_ebuild[:-8] + "4.ebuild")), ("dev-libs/A", RepomanRun(args=["commit", "-m", "cat/pkg: bump to version 4"])), ) env = { "PORTAGE_OVERRIDE_EPREFIX": eprefix, "DISTDIR": distdir, "GENTOO_COMMITTER_NAME": committer_name, "GENTOO_COMMITTER_EMAIL": committer_email, "HOME": homedir, "PATH": os.environ["PATH"], "PORTAGE_GRPNAME": os.environ["PORTAGE_GRPNAME"], "PORTAGE_USERNAME": os.environ["PORTAGE_USERNAME"], "PORTAGE_REPOSITORIES": settings.repositories.config_string(), "PYTHONDONTWRITEBYTECODE": os.environ.get("PYTHONDONTWRITEBYTECODE", ""), } if os.environ.get("SANDBOX_ON") == "1": # avoid problems from nested sandbox instances env["FEATURES"] = "-sandbox -usersandbox" dirs = [homedir, license_dir, profiles_dir, distdir] try: for d in dirs: ensure_dirs(d) with open(os.path.join(test_repo_location, "skel.ChangeLog"), 'w') as f: f.write(copyright_header) with open(os.path.join(profiles_dir, "profiles.desc"), 'w') as f: for x in profiles: f.write("%s %s %s\n" % x) # ResolverPlayground only created the first profile, # so create the remaining ones. for x in profiles[1:]: sub_profile_dir = os.path.join(profiles_dir, x[1]) ensure_dirs(sub_profile_dir) for config_file, lines in profile.items(): file_name = os.path.join(sub_profile_dir, config_file) with open(file_name, "w") as f: for line in lines: f.write("%s\n" % line) for x in licenses: open(os.path.join(license_dir, x), 'wb').close() with open(os.path.join(profiles_dir, "arch.list"), 'w') as f: for x in arch_list: f.write("%s\n" % x) with open(os.path.join(profiles_dir, "use.desc"), 'w') as f: for k, v in use_desc: f.write("%s - %s\n" % (k, v)) for cp, xml_data in metadata_xml_files: with open(os.path.join(test_repo_location, cp, "metadata.xml"), 'w') as f: f.write(playground.metadata_xml_template % xml_data) # Use a symlink to test_repo, in order to trigger bugs # involving canonical vs. non-canonical paths. test_repo_symlink = os.path.join(eroot, "test_repo_symlink") os.symlink(test_repo_location, test_repo_symlink) metadata_xsd_dest = os.path.join( test_repo_location, 'metadata/xml-schema/metadata.xsd') os.makedirs(os.path.dirname(metadata_xsd_dest)) os.symlink(metadata_xsd, metadata_xsd_dest) if debug: # The subprocess inherits both stdout and stderr, for # debugging purposes. stdout = None else: # The subprocess inherits stderr so that any warnings # triggered by python -Wd will be visible. stdout = subprocess.PIPE for cwd in ("", "dev-libs", "dev-libs/A", "dev-libs/B", "dev-libs/C"): abs_cwd = os.path.join(test_repo_symlink, cwd) proc = await asyncio.create_subprocess_exec(*(repoman_cmd + ("full", )), env=env, stderr=None, stdout=stdout, cwd=abs_cwd) if debug: await proc.wait() else: output, _err = await proc.communicate() await proc.wait() if proc.returncode != os.EX_OK: portage.writemsg(output) self.assertEqual(os.EX_OK, proc.returncode, "repoman failed in %s" % (cwd, )) if git_binary is not None: for cwd, cmd in git_test: abs_cwd = os.path.join(test_repo_symlink, cwd) if isinstance(cmd, RepomanRun): cmd.cwd = abs_cwd cmd.env = env cmd.debug = debug await cmd.run() if cmd.result[ "result"] != cmd.expected and cmd.result.get( "stdio"): portage.writemsg(cmd.result["stdio"]) try: self.assertEqual(cmd.result["result"], cmd.expected) except Exception: print(cmd.result["result"], file=sys.stderr, flush=True) raise continue proc = await asyncio.create_subprocess_exec(*cmd, env=env, stderr=None, stdout=stdout, cwd=abs_cwd) if debug: await proc.wait() else: output, _err = await proc.communicate() await proc.wait() if proc.returncode != os.EX_OK: portage.writemsg(output) self.assertEqual( os.EX_OK, proc.returncode, "%s failed in %s" % ( cmd, cwd, ), ) finally: playground.cleanup()
def perform(self, qa_output): myautoadd = self._vcs_autoadd() self._vcs_deleted() changes = self.get_vcs_changed() mynew, mychanged, myremoved, no_expansion, expansion = changes # Manifests need to be regenerated after all other commits, so don't commit # them now even if they have changed. mymanifests = set() myupdates = set() for f in mychanged + mynew: if "Manifest" == os.path.basename(f): mymanifests.add(f) else: myupdates.add(f) myupdates.difference_update(myremoved) myupdates = list(myupdates) mymanifests = list(mymanifests) myheaders = [] commitmessage = self.options.commitmsg if self.options.commitmsgfile: try: f = io.open( _unicode_encode( self.options.commitmsgfile, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') commitmessage = f.read() f.close() del f except (IOError, OSError) as e: if e.errno == errno.ENOENT: portage.writemsg( "!!! File Not Found:" " --commitmsgfile='%s'\n" % self.options.commitmsgfile) else: raise if commitmessage[:9].lower() in ("cat/pkg: ",): commitmessage = self.msg_prefix() + commitmessage[9:] if not commitmessage or not commitmessage.strip(): commitmessage = self.get_new_commit_message(qa_output) commitmessage = commitmessage.rstrip() # Update copyright for new and changed files year = time.strftime('%Y', time.gmtime()) for fn in chain(mynew, mychanged): if fn.endswith('.diff') or fn.endswith('.patch'): continue update_copyright(fn, year, pretend=self.options.pretend) myupdates, broken_changelog_manifests = self.changelogs( myupdates, mymanifests, myremoved, mychanged, myautoadd, mynew, commitmessage) lines = commitmessage.splitlines() lastline = lines[-1] if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None: commitmessage += '\n' commit_footer = self.get_commit_footer() commitmessage += commit_footer print("* %s files being committed..." % green(str(len(myupdates))), end=' ') if not self.vcs_settings.needs_keyword_expansion: # With some VCS types there's never any keyword expansion, so # there's no need to regenerate manifests and all files will be # committed in one big commit at the end. logging.debug("VCS type doesn't need keyword expansion") print() elif not self.repo_settings.repo_config.thin_manifest: logging.debug("perform: Calling thick_manifest()") self.vcs_settings.changes.thick_manifest(myupdates, myheaders, no_expansion, expansion) logging.info("myupdates: %s", myupdates) logging.info("myheaders: %s", myheaders) uq = UserQuery(self.options) if self.options.ask and uq.query('Commit changes?', True) != 'Yes': print("* aborting commit.") sys.exit(128 + signal.SIGINT) # Handle the case where committed files have keywords which # will change and need a priming commit before the Manifest # can be committed. if (myupdates or myremoved) and myheaders: self.priming_commit(myupdates, myremoved, commitmessage) # When files are removed and re-added, the cvs server will put /Attic/ # inside the $Header path. This code detects the problem and corrects it # so that the Manifest will generate correctly. See bug #169500. # Use binary mode in order to avoid potential character encoding issues. self.vcs_settings.changes.clear_attic(myheaders) if self.scanner.repolevel == 1: utilities.repoman_sez( "\"You're rather crazy... " "doing the entire repository.\"\n") self.vcs_settings.changes.digest_regen(myupdates, myremoved, mymanifests, self.scanner, broken_changelog_manifests) if self.repo_settings.sign_manifests: self.sign_manifest(myupdates, myremoved, mymanifests) self.vcs_settings.changes.update_index(mymanifests, myupdates) self.add_manifest(mymanifests, myheaders, myupdates, myremoved, commitmessage) if self.options.quiet: return print() if self.vcs_settings.vcs: print("Commit complete.") else: print( "repoman was too scared" " by not seeing any familiar version control file" " that he forgot to commit anything") utilities.repoman_sez( "\"If everyone were like you, I'd be out of business!\"\n") return
def perform(self, qa_output): myautoadd = self._vcs_autoadd() self._vcs_deleted() changes = self.get_vcs_changed() mynew, mychanged, myremoved, no_expansion, expansion = changes # Manifests need to be regenerated after all other commits, so don't commit # them now even if they have changed. mymanifests = set() myupdates = set() for f in mychanged + mynew: if "Manifest" == os.path.basename(f): mymanifests.add(f) else: myupdates.add(f) myupdates.difference_update(myremoved) myupdates = list(myupdates) mymanifests = list(mymanifests) myheaders = [] commitmessage = self.options.commitmsg if self.options.commitmsgfile: try: f = io.open(_unicode_encode(self.options.commitmsgfile, encoding=_encodings['fs'], errors='strict'), mode='r', encoding=_encodings['content'], errors='replace') commitmessage = f.read() f.close() del f except (IOError, OSError) as e: if e.errno == errno.ENOENT: portage.writemsg("!!! File Not Found:" " --commitmsgfile='%s'\n" % self.options.commitmsgfile) else: raise if commitmessage[:9].lower() in ("cat/pkg: ", ): commitmessage = self.msg_prefix() + commitmessage[9:] if not commitmessage or not commitmessage.strip(): commitmessage = self.get_new_commit_message(qa_output) commitmessage = commitmessage.rstrip() # Update copyright for new and changed files year = time.strftime('%Y', time.gmtime()) for fn in chain(mynew, mychanged): if fn.endswith('.diff') or fn.endswith('.patch'): continue update_copyright(fn, year, pretend=self.options.pretend) myupdates, broken_changelog_manifests = self.changelogs( myupdates, mymanifests, myremoved, mychanged, myautoadd, mynew, commitmessage) lines = commitmessage.splitlines() lastline = lines[-1] if len(lines) == 1 or re.match(r'^\S+:\s', lastline) is None: commitmessage += '\n' commit_footer = self.get_commit_footer() commitmessage += commit_footer print("* %s files being committed..." % green(str(len(myupdates))), end=' ') if not self.vcs_settings.needs_keyword_expansion: # With some VCS types there's never any keyword expansion, so # there's no need to regenerate manifests and all files will be # committed in one big commit at the end. logging.debug("VCS type doesn't need keyword expansion") print() elif not self.repo_settings.repo_config.thin_manifest: logging.debug("perform: Calling thick_manifest()") self.vcs_settings.changes.thick_manifest(myupdates, myheaders, no_expansion, expansion) logging.info("myupdates: %s", myupdates) logging.info("myheaders: %s", myheaders) uq = UserQuery(self.options) if self.options.ask and uq.query('Commit changes?', True) != 'Yes': print("* aborting commit.") sys.exit(128 + signal.SIGINT) # Handle the case where committed files have keywords which # will change and need a priming commit before the Manifest # can be committed. if (myupdates or myremoved) and myheaders: self.priming_commit(myupdates, myremoved, commitmessage) # When files are removed and re-added, the cvs server will put /Attic/ # inside the $Header path. This code detects the problem and corrects it # so that the Manifest will generate correctly. See bug #169500. # Use binary mode in order to avoid potential character encoding issues. self.vcs_settings.changes.clear_attic(myheaders) if self.scanner.repolevel == 1: utilities.repoman_sez("\"You're rather crazy... " "doing the entire repository.\"\n") self.vcs_settings.changes.digest_regen(myupdates, myremoved, mymanifests, self.scanner, broken_changelog_manifests) if self.repo_settings.sign_manifests: self.sign_manifest(myupdates, myremoved, mymanifests) self.vcs_settings.changes.update_index(mymanifests, myupdates) self.add_manifest(mymanifests, myheaders, myupdates, myremoved, commitmessage) if self.options.quiet: return print() if self.vcs_settings.vcs: print("Commit complete.") else: print("repoman was too scared" " by not seeing any familiar version control file" " that he forgot to commit anything") utilities.repoman_sez( "\"If everyone were like you, I'd be out of business!\"\n") return
def perform(self, qa_output): myautoadd = self._vcs_autoadd() self._vcs_deleted() changes = self.get_vcs_changed() mynew, mychanged, myremoved, no_expansion, expansion = changes # Manifests need to be regenerated after all other commits, so don't commit # them now even if they have changed. mymanifests = set() myupdates = set() for f in mychanged + mynew: if "Manifest" == os.path.basename(f): mymanifests.add(f) else: myupdates.add(f) myupdates.difference_update(myremoved) myupdates = list(myupdates) mymanifests = list(mymanifests) myheaders = [] commitmessage = self.options.commitmsg if self.options.commitmsgfile: try: f = io.open( _unicode_encode( self.options.commitmsgfile, encoding=_encodings["fs"], errors="strict", ), mode="r", encoding=_encodings["content"], errors="replace", ) commitmessage = f.read() f.close() del f except (IOError, OSError) as e: if e.errno == errno.ENOENT: portage.writemsg("!!! File Not Found:" " --commitmsgfile='%s'\n" % self.options.commitmsgfile) else: raise if commitmessage[:9].lower() in ("cat/pkg: ", ): commitmessage = self.msg_prefix() + commitmessage[9:] if commitmessage is not None and commitmessage.strip(): res, expl = self.verify_commit_message(commitmessage) if not res: print(bad("RepoMan does not like your commit message:")) print(expl) if self.options.force: print("(but proceeding due to --force)") else: sys.exit(1) else: commitmessage = None msg_qa_output = qa_output initial_message = None while True: commitmessage = self.get_new_commit_message( msg_qa_output, commitmessage) res, expl = self.verify_commit_message(commitmessage) if res: break else: full_expl = ( """Issues with the commit message were found. Please fix it or remove the whole commit message to abort. """ + expl) msg_qa_output = [ " %s\n" % x for x in full_expl.splitlines() ] + qa_output commitmessage = commitmessage.rstrip() # Update copyright for new and changed files year = time.strftime("%Y", time.gmtime()) updated_copyright = [] for fn in chain(mynew, mychanged): if fn.endswith(".diff") or fn.endswith(".patch"): continue if update_copyright(fn, year, pretend=self.options.pretend): updated_copyright.append(fn) if updated_copyright and not ( self.options.pretend or self.repo_settings.repo_config.thin_manifest): for cp in sorted(self._vcs_files_to_cps(iter(updated_copyright))): self._manifest_gen(cp) myupdates, broken_changelog_manifests = self.changelogs( myupdates, mymanifests, myremoved, mychanged, myautoadd, mynew, commitmessage, ) lines = commitmessage.splitlines() lastline = lines[-1] if len(lines) == 1 or re.match(r"^\S+:\s", lastline) is None: commitmessage += "\n" commit_footer = self.get_commit_footer() commitmessage += commit_footer print("* %s files being committed..." % green(str(len(myupdates))), end=" ") if not self.vcs_settings.needs_keyword_expansion: # With some VCS types there's never any keyword expansion, so # there's no need to regenerate manifests and all files will be # committed in one big commit at the end. logging.debug("VCS type doesn't need keyword expansion") print() elif not self.repo_settings.repo_config.thin_manifest: logging.debug("perform: Calling thick_manifest()") self.vcs_settings.changes.thick_manifest(myupdates, myheaders, no_expansion, expansion) logging.info("myupdates: %s", myupdates) logging.info("myheaders: %s", myheaders) uq = UserQuery(self.options) if self.options.ask and uq.query("Commit changes?", True) != "Yes": print("* aborting commit.") sys.exit(128 + signal.SIGINT) # Handle the case where committed files have keywords which # will change and need a priming commit before the Manifest # can be committed. if (myupdates or myremoved) and myheaders: self.priming_commit(myupdates, myremoved, commitmessage) # When files are removed and re-added, the cvs server will put /Attic/ # inside the $Header path. This code detects the problem and corrects it # so that the Manifest will generate correctly. See bug #169500. # Use binary mode in order to avoid potential character encoding issues. self.vcs_settings.changes.clear_attic(myheaders) if self.scanner.repolevel == 1: utilities.repoman_sez("\"You're rather crazy... " 'doing the entire repository."\n') self.vcs_settings.changes.digest_regen(myupdates, myremoved, mymanifests, self.scanner, broken_changelog_manifests) if self.repo_settings.sign_manifests: self.sign_manifest(myupdates, myremoved, mymanifests) self.vcs_settings.changes.update_index(mymanifests, myupdates) self.add_manifest(mymanifests, myheaders, myupdates, myremoved, commitmessage) if self.options.quiet: return print() if self.vcs_settings.vcs: print("Commit complete.") else: print("repoman was too scared" " by not seeing any familiar version control file" " that he forgot to commit anything") utilities.repoman_sez( '"If everyone were like you, I\'d be out of business!"\n') return