def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, submit=False, sort=False, template=None, macros=[], exported=None, create=False, fullnames=False): with open(specfile, encoding='utf-8') as fi: spec, oldchlog = split_spec_changelog(fi) another = None if config.getbool("log", "merge-spec", False): another = oldchlog sort = sort or config.getbool("log", "sort", False) if fullnames: _map_user_names() chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size, submit=submit, sort=sort, template=template, macros=macros, exported=exported, oldlog=True, create=create) with open(specfile, "w", encoding='utf-8') as fo: fo.writelines(spec) fo.write("\n%changelog\n") fo.writelines(chlog)
def specfile_svn2rpm(pkgdirurl, specfile, rev=None, size=None, submit=False, sort=False, template=None, macros=[], exported=None): fi = open(specfile) spec, oldchlog = split_spec_changelog(fi) fi.close() another = None if config.getbool("log", "merge-spec", False): another = oldchlog sort = sort or config.getbool("log", "sort", False) chlog = get_changelog(pkgdirurl, another=another, rev=rev, size=size, submit=submit, sort=sort, template=template, macros=macros, exported=exported, oldlog=True) fo = open(specfile, "w") fo.writelines(spec) fo.write("\n\n%changelog\n") fo.writelines(chlog) fo.close()
def _execVcs(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup") if not kwargs.get("show") and args[0] not in localcmds: args = list(args) args.append("--non-interactive") else: kwargs["geterr"] = True kwargs["cleanerr"] = True cmdstr = self.vcs_command + " " + " ".join(args) try: return execcmd(cmdstr, **kwargs) except Error, e: msg = None if e.args: if "Permission denied" in e.args[0]: msg = ("It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" " for more information.") elif "authorization failed" in e.args[0]: msg = ("Note that repsys does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error, "%s\n%s" % (e, msg) raise
def execcmd(*cmd, **kwargs): cmdstr = " ".join(cmd) if kwargs.get("show"): if kwargs.get("geterr"): err = StringIO() pipe = subprocess.Popen(cmdstr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) of = pipe.stdout.fileno() ef = pipe.stderr.fileno() while True: odata = os.read(of, 8192) sys.stdout.write(odata) edata = os.read(ef, 8192) err.write(edata) sys.stderr.write(edata) status = pipe.poll() if status is not None and not (odata and edata): break output = err.getvalue() else: status = os.system(cmdstr) output = "" else: status, output = commands_getstatusoutput( "LANG=C LANGUAGE=C LC_ALL=C "+cmdstr) verbose = config.getbool("global", "verbose", 0) if status != 0 and not kwargs.get("noerror"): if kwargs.get("cleanerr") and not verbose: raise Error, output else: raise Error, "command failed: %s\n%s\n" % (cmdstr, output) if verbose: print cmdstr sys.stdout.write(output) return status, output
def _execVcs(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup") if not kwargs.get("show") and args[0] not in localcmds: args = list(args) args.append("--non-interactive") else: kwargs["geterr"] = True kwargs["cleanerr"] = True cmdstr = self.vcs_command + " " + " ".join(args) try: return execcmd(cmdstr, **kwargs) except Error, e: msg = None if e.args: if "Permission denied" in e.args[0]: msg = ( "It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" " for more information.") elif "authorization failed" in e.args[0]: msg = ("Note that repsys does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error, "%s\n%s" % (e, msg) raise
def repository_url(mirrored=False): url = None if mirrored and config.getbool("global", "use-mirror", "yes"): url = config.get("global", "mirror") if url is None: url = config.get("global", "repository") if not url: # compatibility with the default_parent configuration option default_parent = config.get("global", "default_parent") if default_parent is None: raise Error, "you need to set the 'repository' " \ "configuration option on repsys.conf" url = convert_default_parent(default_parent) return url
def _execVcs(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup", "mv") cmd = self.vcs_command + list(args) if self.env_defaults: self._set_env() kwargs["collecterr"] = kwargs.get("collecterr", False) if kwargs.get("show"): if not kwargs.get("local"): kwargs["collecterr"] = True else: if self.vcs_command is "svn" and args[0] not in localcmds: cmd.append("--non-interactive") else: if args[0] == "mv": kwargs["collecterr"] = False kwargs["cleanerr"] = kwargs.get("cleanerr", True) if kwargs.get("xml"): cmd.append("--xml") try: if args[0] in ('info', 'checkout', 'log'): kwargs['info'] = True else: kwargs['info'] = False return execcmd(*cmd, **kwargs) except Error as e: msg = None if e.args: if "Permission denied" in e.args: msg = ("It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "https://wiki.mageia.org/en/Packagers_ssh" " for more information.") elif "authorization failed" in e.args: msg = ("Note that repsys does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error("%s\n%s" % (e, msg)) raise
def execcmd(*cmd, **kwargs): cmdstr = " ".join(cmd) if kwargs.get("show"): if kwargs.get("geterr"): err = StringIO() pipe = subprocess.Popen(cmdstr, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) of = pipe.stdout.fileno() ef = pipe.stderr.fileno() while True: odata = os.read(of, 8192) sys.stdout.write(odata) edata = os.read(ef, 8192) err.write(edata) sys.stderr.write(edata) status = pipe.poll() if status is not None and not (odata and edata): break output = err.getvalue() else: status = os.system(cmdstr) output = "" else: status, output = commands_getstatusoutput( "LANG=C LANGUAGE=C LC_ALL=C " + cmdstr) verbose = config.getbool("global", "verbose", 0) if status != 0 and not kwargs.get("noerror"): if kwargs.get("cleanerr") and not verbose: raise Error, output else: raise Error, "command failed: %s\n%s\n" % (cmdstr, output) if verbose: print cmdstr sys.stdout.write(output) return status, output
def get_srpm(pkgdirurl, mode="current", targetdirs=None, version=None, release=None, revision=None, packager="", revname=0, svnlog=0, scripts=[], submit=False, template=None, distro=None, macros=[], verbose=0, strict=False): svn = SVN() tmpdir = tempfile.mktemp() topdir = "_topdir %s" % tmpdir builddir = "_builddir %s/%s" % (tmpdir, "BUILD") rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS") sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES") specdir = "_specdir %s/%s" % (tmpdir, "SPECS") srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS") patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES") temppath = "_tmppath %s" % (tmpdir) rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir, sourcedir, specdir, srcrpmdir, patchdir, temppath)] try: if mode == "version": geturl = layout.checkout_url(pkgdirurl, version=version, release=release) elif mode == "pristine": geturl = layout.checkout_url(pkgdirurl, pristine=True) elif mode == "current" or mode == "revision": #FIXME we should handle revisions specified using @REV geturl = layout.checkout_url(pkgdirurl) else: raise Error, "unsupported get_srpm mode: %s" % mode strict = strict or config.getbool("submit", "strict-revision", False) if strict and not rev_touched_url(geturl, revision): #FIXME would be nice to have the revision number even when # revision is None raise Error, "the revision %s does not change anything "\ "inside %s" % (revision or "HEAD", geturl) mirror.info(geturl) svn.export(geturl, tmpdir, rev=revision) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) if config.getbool("srpm", "run-prep", False): makefile = os.path.join(tmpdir, "Makefile") if os.path.exists(makefile): execcmd(("make", "-C", tmpdir, "srpm-prep")) if not speclist: raise Error, "no spec files found" spec = speclist[0] if svnlog: submit = not not revision log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir) for script in scripts: #FIXME revision can be "None" status, output = execcmd(script, tmpdir, spec, str(revision), noerror=1) if status != 0: raise Error, "script %s failed" % script if packager: packager = " --define 'packager %s'" % packager if distro: cmpdistro = distro.lower() for target in cgiutil.get_targets(): if target.name.lower() == cmpdistro: macros.extend(target.macros) break else: raise Error, "no such submit target in configuration: %s" % ( distro) sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") args = [sourcecmd, "-bs", "--nodeps"] for pair in rpmdefs: args.extend(pair) for pair in macros: args.extend(("--define", "%s %s" % pair)) args.append(spec) try: execcmd(args) except CommandError, e: if config.getbool("global", "verbose"): cmdline = e.cmdline + "\n" else: cmdline = "" raise Error, ("error while creating the source RPM " "(with %s):\n%s%s" % (sourcecmd, cmdline, e.output)) # copy the generated SRPMs to their target locations targetsrpms = [] urlrev = None if revname: urlrev = revision or layout.get_url_revision(geturl) if not targetdirs: targetdirs = (".", ) srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm")) if not srpms: # something fishy happened raise Error, "no SRPMS were found at %s" % srpmsdir for srpm in srpms: name = os.path.basename(srpm) if revname: name = "@%s:%s" % (urlrev, name) for targetdir in targetdirs: newpath = os.path.join(targetdir, name) targetsrpms.append(newpath) if os.path.exists(newpath): # should we warn? os.unlink(newpath) shutil.copy(srpm, newpath) if verbose: sys.stderr.write("Wrote: %s\n" % newpath) return targetsrpms
def get_srpm(pkgdirurl, mode = "current", targetdirs = None, version = None, release = None, revision = None, packager = "", revname = 0, svnlog = 0, scripts = [], submit = False, template = None, distro = None, macros = [], verbose = 0, strict = False): svn = SVN() tmpdir = tempfile.mktemp() topdir = "_topdir %s" % tmpdir builddir = "_builddir %s/%s" % (tmpdir, "BUILD") rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS") sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES") specdir = "_specdir %s/%s" % (tmpdir, "SPECS") srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS") patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES") temppath = "_tmppath %s" % (tmpdir) rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir, sourcedir, specdir, srcrpmdir, patchdir, temppath)] try: if mode == "version": geturl = layout.checkout_url(pkgdirurl, version=version, release=release) elif mode == "pristine": geturl = layout.checkout_url(pkgdirurl, pristine=True) elif mode == "current" or mode == "revision": #FIXME we should handle revisions specified using @REV geturl = layout.checkout_url(pkgdirurl) else: raise Error, "unsupported get_srpm mode: %s" % mode strict = strict or config.getbool("submit", "strict-revision", False) if strict and not rev_touched_url(geturl, revision): #FIXME would be nice to have the revision number even when # revision is None raise Error, "the revision %s does not change anything "\ "inside %s" % (revision or "HEAD", geturl) mirror.info(geturl) svn.export(geturl, tmpdir, rev=revision) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) if config.getbool("srpm", "run-prep", False): makefile = os.path.join(tmpdir, "Makefile") if os.path.exists(makefile): execcmd(("make", "-C", tmpdir, "srpm-prep")) if not speclist: raise Error, "no spec files found" spec = speclist[0] if svnlog: submit = not not revision log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir) for script in scripts: #FIXME revision can be "None" status, output = execcmd(script, tmpdir, spec, str(revision), noerror=1) if status != 0: raise Error, "script %s failed" % script if packager: packager = " --define 'packager %s'" % packager if distro: cmpdistro = distro.lower() for target in cgiutil.get_targets(): if target.name.lower() == cmpdistro: macros.extend(target.macros) break else: raise Error, "no such submit target in configuration: %s" % (distro) sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") args = [sourcecmd, "-bs", "--nodeps"] for pair in rpmdefs: args.extend(pair) for pair in macros: args.extend(("--define", "%s %s" % pair)) args.append(spec) try: execcmd(args) except CommandError, e: if config.getbool("global", "verbose"): cmdline = e.cmdline + "\n" else: cmdline = "" raise Error, ("error while creating the source RPM " "(with %s):\n%s%s" % (sourcecmd, cmdline, e.output)) # copy the generated SRPMs to their target locations targetsrpms = [] urlrev = None if revname: urlrev = revision or layout.get_url_revision(geturl) if not targetdirs: targetdirs = (".",) srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm")) if not srpms: # something fishy happened raise Error, "no SRPMS were found at %s" % srpmsdir for srpm in srpms: name = os.path.basename(srpm) if revname: name = "@%s:%s" % (urlrev, name) for targetdir in targetdirs: newpath = os.path.join(targetdir, name) targetsrpms.append(newpath) if os.path.exists(newpath): # should we warn? os.unlink(newpath) shutil.copy(srpm, newpath) if verbose: sys.stderr.write("Wrote: %s\n" % newpath) return targetsrpms
def get_srpm(pkgdirurl, mode="current", targetdirs=None, version=None, release=None, revision=None, packager="", revname=0, svnlog=0, scripts=[], submit=False, template=None, macros=[], verbose=0, strict=False, fullnames=False): svn = detectVCS(pkgdirurl) tmpdir = tempfile.mktemp() topdir = "_topdir %s" % tmpdir builddir = "_builddir %s/%s" % (tmpdir, "BUILD") rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS") sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES") specdir = "_specdir %s/%s" % (tmpdir, "SPECS") srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS") patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES") temppath = "_tmppath %s" % (tmpdir) rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir, sourcedir, specdir, srcrpmdir, patchdir, temppath)] try: if mode == "version": geturl = layout.checkout_url(pkgdirurl, version=version, release=release) elif mode == "pristine": geturl = layout.checkout_url(pkgdirurl, pristine=True) elif mode == "current" or mode == "revision": #FIXME we should handle revisions specified using @REV geturl = layout.checkout_url(pkgdirurl) else: raise Error("unsupported get_srpm mode: %s" % mode) strict = strict or config.getbool("submit", "strict-revision", False) if strict and not rev_touched_url(geturl, revision): #FIXME would be nice to have the revision number even when # revision is None raise Error("the revision %s does not change anything "\ "inside %s" % (revision or "HEAD", geturl)) mirror.info(geturl) svn.export(geturl, tmpdir, rev=revision) binrepo.download_binaries(tmpdir) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) if config.getbool("srpm", "run-prep", False): makefile = os.path.join(tmpdir, "Makefile") if os.path.exists(makefile): execcmd(("make", "-C", tmpdir, "srpm-prep")) if not speclist: raise Error("no spec files found") spec = speclist[0] sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") if packager: packager = " --define 'packager %s'" % packager sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") args = [sourcecmd, "-bs", "--nodeps"] for pair in rpmdefs: args.extend(pair) for pair in macros: args.extend(("--define", "%s %s" % pair)) args.append(spec) if svnlog: submit = not not revision try: log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir, fullnames=fullnames) except: #cmd = [sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir execcmd(args) cp_srpms(revision, revname, geturl, targetdirs, srpmsdir, verbose) log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir, create=True) for script in scripts: #FIXME revision can be "None" status, output = execcmd(script, tmpdir, spec, str(revision), noerror=1) if status != 0: raise Error("script %s failed" % script) try: execcmd(args) except CommandError as e: if config.getbool("global", "verbose"): cmdline = e.cmdline + "\n" else: cmdline = "" raise Error("error while creating the source RPM " "(with %s):\n%s%s" % (sourcecmd, cmdline, e.output)) # copy the generated SRPMs to their target locations targetsrpms = cp_srpms(revision, revname, geturl, targetdirs, srpmsdir, verbose) return targetsrpms finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir)
def execcmd(*cmd, **kwargs): assert (kwargs.get("collecterr") and kwargs.get("show")) or not kwargs.get("collecterr"), \ ("execcmd is implemented to handle collecterr=True only if show=True") # split command args if isinstance(cmd[0], str): if len(cmd) is 1: cmdargs = shlex.split(cmd[0]) else: cmdargs = cmd[:] else: cmdargs = cmd[0][:] stdout = None stderr = None env = {} env.update(os.environ) if kwargs.get("info") or not kwargs.get("show") or ( kwargs.get("show") and kwargs.get("collecterr")): if kwargs.get("info"): env.update({"LANGUAGE": "C", "LC_TIME": "C"}) else: env.update({"LANG": "C", "LANGUAGE": "C", "LC_ALL": "C"}) stdout = subprocess.PIPE if kwargs.get("collecterr"): stderr = subprocess.PIPE else: stderr = subprocess.STDOUT verbose = config.getbool("global", "verbose", 0) if verbose: print("cmd: " + str(cmd).lstrip("(").rstrip(")").replace("', '", "' '")) proc = subprocess.Popen(cmdargs, shell=False, stdout=stdout, stdin=kwargs.get("stdin"), stderr=stderr, env=env) output = "" if kwargs.get("show") and kwargs.get("collecterr"): error = StringIO() wl = [] outfd = proc.stdout.fileno() errfd = proc.stderr.fileno() rl = [outfd, errfd] xl = wl while proc.poll() is None: mrl, _, _ = select.select(rl, wl, xl, 0.5) for fd in mrl: data = os.read(fd, 8192).decode('utf8') if fd == errfd: error.write(data) sys.stderr.write(data) else: sys.stdout.write(data) output = error.getvalue() else: # Make sure to avoid buffer getting full. # Otherwise if ie. using proc.wait() both python # and the process will just hang while proc.poll() is None: if proc.stdout is not None: output += proc.stdout.read(8192).decode('utf8') # Make sure that we've emptied the buffer entirely if proc.stdout is not None: output += proc.stdout.read().decode('utf8') if kwargs.get("strip", True): output = output.rstrip() if (not kwargs.get("noerror")) and proc.returncode != 0: if kwargs.get("cleanerr"): msg = output cmdline = subprocess.list2cmdline(cmdargs) raise CommandError(cmdline, proc.returncode, output) return proc.returncode, output