def maintdb(maintdb_args): host = config.get("maintdb", "host", "maintdb.mageia.org") maintdb_helper = get_helper("maintdb") cmd_args = ' '.join(maintdb_args) command = "ssh %s %s %s" % (host, maintdb_helper, cmd_args) execcmd(command, show=True) sys.exit(0)
def list_targets(option, opt, val, parser): host = config.get("submit", "host") if host is None: raise Error, "no submit host defined in mgarepo.conf" createsrpm = get_helper("create-srpm") #TODO make it configurable command = "ssh %s %s --list" % (host, createsrpm) execcmd(command, show=True) sys.exit(0)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None): mirror.info(pkgdirurl) url = checkout_url(pkgdirurl) svncmd = config.get("global", "svn-command", "svn") args = [svncmd, "log", url] if verbose: args.append("-v") if limit: args.append("-l") args.append(limit) if revision: args.append("-r") args.append(revision) if os.isatty(sys.stdin.fileno()): args.append("| less") rawcmd = " ".join(args) execcmd(rawcmd, show=True)
def _execsvnlook(self, cmd, *args, **kwargs): execcmd_args = ["svnlook", cmd, self.repospath] self._add_txnrev(execcmd_args, kwargs) execcmd_args += args execcmd_kwargs = {} keywords = ["show", "noerror"] for key in keywords: if kwargs.has_key(key): execcmd_kwargs[key] = kwargs[key] return execcmd(*execcmd_args, **execcmd_kwargs)
def patch_spec(pkgdirurl, patchfile, log=""): #FIXME use get_spec svn = SVN() tmpdir = tempfile.mktemp() try: geturl = layout.checkout_url(pkgdirurl, append_path="SPECS") svn.checkout(geturl, tmpdir) speclist = glob.glob(os.path.join(tmpdir, "*.spec")) if not speclist: raise Error, "no spec files found" spec = speclist[0] status, output = execcmd("patch", spec, patchfile) if status != 0: raise Error, "can't apply patch:\n%s\n" % output else: svn.commit(tmpdir, log="") finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): if submithost is None: submithost = config.get("submit", "host") if submithost is None: # extract the submit host from the svn host type, rest = urllib.splittype(pkgdirurl) host, path = urllib.splithost(rest) user, host = urllib.splituser(host) submithost, port = urllib.splitport(host) del type, user, port, path, rest # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory createsrpm = get_helper("create-srpm") baseargs = ["ssh", submithost, createsrpm, "-t", target] if not sid: sid = uuid.uuid4() define.append("sid=%s" % sid) for entry in reversed(define): baseargs.append("--define") baseargs.append(entry) cmdsargs = [] if len(urls) == 1: # be compatible with server-side mgarepo versions older than 1.6.90 url, rev = layout.split_url_revision(urls[0]) baseargs.append("-r") baseargs.append(str(rev)) baseargs.append(url) cmdsargs.append(baseargs) elif atonce: cmdsargs.append(baseargs + urls) else: cmdsargs.extend((baseargs + [url]) for url in urls) for cmdargs in cmdsargs: command = subprocess.list2cmdline(cmdargs) status, output = execcmd(command) if status == 0: print "Package submitted!" else: sys.stderr.write(output) sys.exit(status)
def _execsvn(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup") if not kwargs.get("show") and args[0] not in localcmds: args = list(args) args.append("--non-interactive") else: kwargs["geterr"] = True kwargs["cleanerr"] = True if kwargs.get("xml"): args.append("--xml") self._set_env() svn_command = config.get("global", "svn-command", "svn") cmdstr = svn_command + " " + " ".join(args) try: return execcmd(cmdstr, **kwargs) except Error, e: msg = None if e.args: if "Permission denied" in e.args[0]: msg = ("It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "https://wiki.mageia.org/en/Packagers_ssh" " for more information.") elif "authorization failed" in e.args[0]: msg = ("Note that mgarepo does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error, "%s\n%s" % (e, msg) raise
def _getinfo(self): cmdstr = "rpm -qp --nosignature --qf '%%{name} %%{epoch} %%{release} %%{version}' %s" status, output = execcmd(cmdstr % self.filename) self.name, self.epoch, self.release, self.version = output.split() if self.epoch == "(none)": self.epoch = None
def unpack(self, topdir): execcmd(("rpm -i --nodeps --define '_sourcedir %s/SOURCES' " + "--define '_specdir %s/SPECS' --define '_patchdir %s/SOURCES' %s") % (topdir, topdir, topdir, self.filename))
"http://binrepo.mageia.org/") url = mirror.normalize_path(url + "/" + sha1) dest = os.path.join(topdir, 'SOURCES', filename) if os.path.exists(dest): if file_hash(dest) == sha1: return 1 else: raise Error, "File with incorrect sha1sum: %s" % dest context = {"dest": dest, "url": url} try: cmd = string.Template(fmt).substitute(context) except KeyError, e: raise Error, "invalid variable %r in download-command "\ "configuration option" % e try: status, output = execcmd(cmd, show=True) except Error, e: os.unlink(dest) raise Error, "Could not download file %s\n" % url def download_binaries(topdir): spath = sources_path(topdir) if not os.path.exists(spath): raise Error, "'%s' was not found" % spath entries = parse_sources(spath) for name, sha1 in entries.iteritems(): download_binary(topdir, sha1, name) def binary_exists(sha1sum): dlurl = config.get("binrepo", "download_url", "http://binrepo.mageia.org/")
def get_srpm(pkgdirurl, mode = "current", targetdirs = None, version = None, release = None, revision = None, packager = "", revname = 0, svnlog = 0, scripts = [], submit = False, template = None, macros = [], verbose = 0, strict = False): svn = SVN() tmpdir = tempfile.mktemp() topdir = "--define '_topdir %s'" % tmpdir builddir = "--define '_builddir %s/%s'" % (tmpdir, "BUILD") rpmdir = "--define '_rpmdir %s/%s'" % (tmpdir, "RPMS") sourcedir = "--define '_sourcedir %s/%s'" % (tmpdir, "SOURCES") specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS") srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS") patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES") try: if mode == "version": geturl = layout.checkout_url(pkgdirurl, version=version, release=release) elif mode == "pristine": geturl = layout.checkout_url(pkgdirurl, pristine=True) elif mode == "current" or mode == "revision": #FIXME we should handle revisions specified using @REV geturl = layout.checkout_url(pkgdirurl) else: raise Error, "unsupported get_srpm mode: %s" % mode strict = strict or config.getbool("submit", "strict-revision", False) if strict and not rev_touched_url(geturl, revision): #FIXME would be nice to have the revision number even when # revision is None raise Error, "the revision %s does not change anything "\ "inside %s" % (revision or "HEAD", geturl) mirror.info(geturl) svn.export(geturl, tmpdir, rev=revision) binrepo.download_binaries(tmpdir) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) if config.getbool("srpm", "run-prep", False): makefile = os.path.join(tmpdir, "Makefile") if os.path.exists(makefile): execcmd("make", "-C", tmpdir, "srpm-prep") if not speclist: raise Error, "no spec files found" spec = speclist[0] if svnlog: submit = not not revision log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir) for script in scripts: #FIXME revision can be "None" status, output = execcmd(script, tmpdir, spec, str(revision), noerror=1) if status != 0: raise Error, "script %s failed" % script if packager: packager = " --define 'packager %s'" % packager defs = rpm_macros_defs(macros) sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" % (sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir, srcrpmdir, patchdir, packager, spec, defs)) # copy the generated SRPMs to their target locations targetsrpms = [] urlrev = None if revname: urlrev = revision or layout.get_url_revision(geturl) if not targetdirs: targetdirs = (".",) srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm")) if not srpms: # something fishy happened raise Error, "no SRPMS were found at %s" % srpmsdir for srpm in srpms: name = os.path.basename(srpm) if revname: name = "@%s:%s" % (urlrev, name) for targetdir in targetdirs: newpath = os.path.join(targetdir, name) targetsrpms.append(newpath) if os.path.exists(newpath): # should we warn? os.unlink(newpath) shutil.copy(srpm, newpath) if verbose: sys.stderr.write("Wrote: %s\n" % newpath) return targetsrpms finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir)
def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, baseurl=None, baseold=None, logmsg=None, rename=True): svn = SVN() srpm = SRPM(srpmfile) tmpdir = tempfile.mktemp() if baseurl: pkgurl = mirror._joinurl(baseurl, srpm.name) else: pkgurl = layout.package_url(srpm.name, distro=branch, mirrored=False) print "Importing package to %s" % pkgurl try: if srpm.epoch: version = "%s:%s" % (srpm.epoch, srpm.version) else: version = srpm.version versionurl = "/".join([pkgurl, "releases", version]) releaseurl = "/".join([versionurl, srpm.release]) currenturl = "/".join([pkgurl, "current"]) currentdir = os.path.join(tmpdir, "current") ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory") if ret or not svn.ls(currenturl, noerror=1): svn.checkout(pkgurl, tmpdir) svn.mkdir(os.path.join(tmpdir, "releases")) svn.mkdir(currentdir) svn.mkdir(os.path.join(currentdir, "SPECS")) svn.mkdir(os.path.join(currentdir, "SOURCES")) #svn.commit(tmpdir,log="Created package structure.") version_exists = 1 else: raise Error, "package already exists or error creating package directory" specsdir = os.path.join(currentdir, "SPECS") sourcesdir = os.path.join(currentdir, "SOURCES") unpackdir = tempfile.mktemp() os.mkdir(unpackdir) try: srpm.unpack(unpackdir) uspecsdir = os.path.join(unpackdir, "SPECS") usourcesdir = os.path.join(unpackdir, "SOURCES") uspecsentries = os.listdir(uspecsdir) usourcesentries = os.listdir(usourcesdir) specsentries = os.listdir(specsdir) sourcesentries = os.listdir(sourcesdir) # Remove old entries for entry in [x for x in specsentries if x not in uspecsentries]: if entry == ".svn": continue entrypath = os.path.join(specsdir, entry) os.unlink(entrypath) svn.remove(entrypath) for entry in [x for x in sourcesentries if x not in usourcesentries]: if entry == ".svn": continue entrypath = os.path.join(sourcesdir, entry) os.unlink(entrypath) svn.remove(entrypath) # Copy all files execcmd("cp -rf", uspecsdir, currentdir) execcmd("cp -rf", usourcesdir, currentdir) # Add new entries for entry in [x for x in uspecsentries if x not in specsentries]: entrypath = os.path.join(specsdir, entry) svn.add(entrypath) for entry in [x for x in usourcesentries if x not in sourcesentries]: entrypath = os.path.join(sourcesdir, entry) if binrepo.is_binary(entrypath): continue svn.add(entrypath) finally: if os.path.isdir(unpackdir): shutil.rmtree(unpackdir) specs = glob.glob(os.path.join(specsdir, "*.spec")) if not specs: raise Error, "no spec file found on %s" % specsdir if len(specs) > 1: raise Error, "more than one spec file found on %s" % specsdir specpath = specs[0] if rename: specfile = os.path.basename(specpath) specname = specfile[:-len(".spec")] if specname != srpm.name: newname = srpm.name + ".spec" newpath = os.path.join(specsdir, newname) sys.stderr.write("warning: renaming spec file to '%s' " "(use -n to disable it)\n" % (newname)) os.rename(specpath, newpath) try: svn.remove(specpath) except Error: # file not tracked svn.revert(specpath) svn.add(newpath) specpath = newpath if striplog: specpath = specpath fspec = open(specpath) spec, chlog = log.split_spec_changelog(fspec) fspec.close() fspec = open(specpath, "w") fspec.writelines(spec) fspec.close() chlog.seek(0, os.SEEK_END) if chlog.tell() != 0: chlog.seek(0) #FIXME move it to layout.py oldurl = baseold or config.get("log", "oldurl") if oldurl == '.' or oldurl.startswith('./'): pkgoldurl = os.path.join(pkgurl, oldurl) else: pkgoldurl = mirror._joinurl(oldurl, srpm.name) svn.mkdir(pkgoldurl, noerror=1, log="created old log directory for %s" % srpm.name) logtmp = tempfile.mktemp() try: svn.checkout(pkgoldurl, logtmp) miscpath = os.path.join(logtmp, "log") fmisc = open(miscpath, "w+") fmisc.writelines(chlog) fmisc.close() svn.add(miscpath) svn.commit(logtmp, log="imported old log for %s" % srpm.name) finally: if os.path.isdir(logtmp): shutil.rmtree(logtmp) binrepo.import_binaries(currentdir, srpm.name) svn.commit(tmpdir, log=logmsg or ("imported package %s" % srpm.name)) finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) # Do revision and pristine tag copies pristineurl = layout.checkout_url(pkgurl, pristine=True) svn.remove(pristineurl, noerror=1, log="Removing previous pristine/ directory.") currenturl = layout.checkout_url(pkgurl) svn.copy(currenturl, pristineurl, log="Copying release %s-%s to pristine/ directory." % (version, srpm.release)) if markrelease: svn.copy(currenturl, releaseurl, log="Copying release %s-%s to releases/ directory." % (version, srpm.release))
if not (source in binrepoentries) and sourcesst.get(source)[1] == '?': toadd.append(sourcepath) elif not download and not os.path.isfile(sourcepath): sys.stderr.write("warning: %s not found\n" % sourcepath) elif download and not os.path.isfile(sourcepath): print "%s not found, downloading from %s" % (sourcepath, url) fmt = config.get("global", "download-command", "wget -c -O '$dest' $url") context = {"dest": sourcepath, "url": url} try: cmd = string.Template(fmt).substitute(context) except KeyError, e: raise Error, "invalid variable %r in download-command "\ "configuration option" % e try: status, output = execcmd(cmd, show=True) except Error, e: os.unlink(sourcepath) raise Error, "Could not download file %s\n" % url if os.path.isfile(sourcepath): toadd.append(sourcepath) else: raise Error, "file not found: %s" % sourcepath # rm entries not found in sources and still in svn found = os.listdir(sourcesdir) for entry in found: if entry == ".svn" or entry == binrepo.SOURCES_FILE: continue status = sourcesst.get(entry) path = os.path.join(sourcesdir, entry)