def layout_dirs(): devel_branch = config.get("global", "trunk-dir", "cauldron/") devel_branch = os.path.normpath(devel_branch) branches_dir = config.get("global", "branches-dir", "updates/") branches_dir = os.path.normpath(branches_dir) backports_dir = config.get("global", "backports-dir", "backports/") backports_dir = os.path.normpath(backports_dir) obsolete_dir = config.get("global", "obsolete-dir", "obsolete/") obsolete_dir = os.path.normpath(obsolete_dir) return devel_branch, branches_dir, backports_dir, obsolete_dir
def get_helper(name): """Tries to find the path of a helper script It first looks if the helper has been explicitly defined in configuration, if not, falls back to the default helper path, which can also be defined in configuration file(s). """ helperdir = config.get("helper", "prefix", "/usr/share/repsys") hpath = config.get("helper", name, None) or \ os.path.join(helperdir, name) return hpath
def get_helper(name): """Tries to find the path of a helper script It first looks if the helper has been explicitly defined in configuration, if not, falls back to the default helper path, which can also be defined in configuration file(s). """ helperdir = config.get("helper", "prefix", "/usr/share/repsys") hpath = config.get("helper", name, None) or \ os.path.join(helperdir, name) if not os.path.isfile(hpath): log.warn("providing unexistent helper: %s", hpath) return hpath
def repository_url(mirrored=False): url = None if mirrored and config.getbool("global", "use-mirror", "yes"): url = config.get("global", "mirror") if url is None: url = config.get("global", "repository") if not url: # compatibility with the default_parent configuration option default_parent = config.get("global", "default_parent") if default_parent is None: raise Error, "you need to set the 'repository' " \ "configuration option on repsys.conf" url = convert_default_parent(default_parent) return url
def submit_package(self, packageurl, packagerev, targetname): username = os.environ.get("REMOTE_USER") packager = config.get("users", username) if not packager: raise CgiError, "your email was not found" elif not packagerev: raise CgiError, "no revision provided" elif not targetname: raise CgiError, "no target provided" else: targetname = targetname.lower() for target in get_targets(): if target.name.lower() == targetname: break else: raise CgiError, "target not found" try: tmp = int(packagerev) except ValueError: raise CgiError, "invalid revision provided" for allowed in target.allowed: if packageurl.startswith(allowed): break else: raise CgiError, "%s is not allowed for this target" \ % packageurl get_srpm(packageurl, revision=packagerev, targetdirs=target.target, packager=packager, revname=1, svnlog=1, scripts=target.scripts) return 1
def download_sources(self): f = open(join(self._git.path, "sources")) downloader = config.get( "global", "download-command", "wget -c -O '$dest' $url").strip("'$dest' $url").split() for line in f.readlines(): fields = line.split() binurl = self.binrepo + "/" + self._package + "/" if len(fields) == 2: checksum, source = fields binurl += source + "/" else: pattern = re.compile(r'^(.*) \((.*)\) = (.*)') match = pattern.match(line) hashtype = match.group(1) source = match.group(2) checksum = match.group(3) binurl += source + "/" + hashtype.lower() + "/" binurl += checksum + "/" + source cmd = downloader + [join(self._git.path, source), binurl] f.close() status, output = execcmd(cmd, show=True) if status == 0: return True else: raise Error("Failed downloading %s, retcode: %d err: %s\n", status, output)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None, releases=None): mirror.info(pkgdirurl) url = checkout_url(pkgdirurl, releases=releases) svncmd = config.get("global", "svn-command", "svn") args = [svncmd, "log", url] if verbose: args.append("-v") if limit: args.append("-l") args.append(limit) if revision: args.append("-r") args.append(revision) if os.isatty(sys.stdin.fileno()): pager = shlex.split(os.environ.get("PAGER", "less")) p = subprocess.Popen(args, stdout=subprocess.PIPE) p2 = subprocess.Popen(pager, stdin=p.stdout) p2.wait() p.wait() else: execcmd(args, show=True)
def get_old_log(pkgdirurl): chlog = StringIO() oldurl = config.get("log", "oldurl") if oldurl: svn = SVN(url=oldurl) tmpdir = tempfile.mktemp() try: if oldurl == '.' or oldurl.startswith('./'): pkgoldurl = os.path.join(pkgdirurl, oldurl) else: pkgname = layout.package_name(pkgdirurl) pkgoldurl = os.path.join(svn.url, pkgname) try: # we're using HEAD here because fixes in misc/ (oldurl) may # be newer than packages' last changed revision. svn.export(pkgoldurl, tmpdir) except Error: pass else: logfile = os.path.join(tmpdir, "log") if os.path.isfile(logfile): with open(logfile, 'r', encoding='utf-8') as lf: chlog.write("\n") # TODO needed? log = lf.read() log = escape_macros(log) chlog.write(log) finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) chlog.seek(0) return chlog
def get_old_log(pkgdirurl): chlog = StringIO() oldurl = config.get("log", "oldurl") if oldurl: svn = SVN() tmpdir = tempfile.mktemp() try: pkgname = layout.package_name(pkgdirurl) pkgoldurl = os.path.join(oldurl, pkgname) try: # we're using HEAD here because fixes in misc/ (oldurl) may # be newer than packages' last changed revision. svn.export(pkgoldurl, tmpdir) except Error: pass else: logfile = os.path.join(tmpdir, "log") if os.path.isfile(logfile): file = open(logfile) chlog.write("\n") # TODO needed? log = file.read() log = escape_macros(log) chlog.write(log) file.close() finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) chlog.seek(0) return chlog
def get_author_name(author): found = emailpat.match(config.get("users", author, author)) gold = emailpat.match(usermap.get(author, "")) name = ((found and found.group("name")) or (gold and gold.group("name")) or author) email = ((found and found.group("email")) or (gold and gold.group("email")) or author + "@mageia.org") return name, email
def get_repsys_cmd_path(cmd): repsys = config.get("global", "repsys-cmd") if repsys: dir = os.path.dirname(repsys) path = os.path.join(dir, cmd) if os.path.exists(path): return path return cmd
def maintdb_get(package): dlurl = config.get("maintdb", "url", "http://maintdb.mageia.org/") dlurl = dlurl + "/" + package h = httplib2.Http() resp, content = h.request(dlurl, 'GET') if resp.status != 200: raise Exception('Package cannot be found in maintdb') return content.rstrip('\n')
def __init__(self, package): self._package = package self._login = config.get("fedora", "login") if self._login: self._repourl = "ssh://" + self._login + "@pkgs.fedoraproject.org/rpms/" + self._package else: self._repourl = "https://src.fedoraproject.org/git/rpms/" + self._package + GIT.vcs_dirname self._git = GIT(url=self._repourl)
def upload(paths, commit=False): topdir = getpkgtopdir() svn = detectVCS(topdir) for path in paths: if os.path.isdir(path) or binrepo.is_binary(path): binrepo.upload_binary(topdir, os.path.basename(path)) binrepo.update_sources(topdir, added=[path]) if commit: silent = config.get("log", "ignore-string", "SILENT") message = "%s: new file %s" % (silent, path) svn.commit(binrepo.sources_path(topdir), log=message) else: svn.add(path, local=True) if commit: silent = config.get("log", "ignore-string", "SILENT") message = "%s: new file %s" % (silent, path) svn.commit(path, log=message)
def __init__(self, path=None, url=None): VCS.__init__(self, path, url) vcs = getattr(VCS, "vcs") vcs.append((self.vcs_name, self.vcs_dirname)) setattr(VCS, "vcs", vcs) self.vcs_command = config.get("global", "git-command", ["git"]) self.vcs_supports['clone'] = True self.env_defaults = {"GIT_SSH": self.vcs_wrapper}
def list_targets(option, opt, val, parser): host = config.get("submit", "host") if host is None: raise Error, "no submit host defined in repsys.conf" createsrpm = get_helper("create-srpm") #TODO make it configurable args = ["ssh", host, createsrpm, "--list"] execcmd(args, show=True) sys.exit(0) # it is invoked via optparse callback, thus we need to
def list_targets(option, opt, val, parser): host = config.get("submit", "host") if host is None: raise Error, "no submit host defined in repsys.conf" createsrpm = get_helper("create-srpm") #TODO make it configurable command = "ssh %s %s --list" % (host, createsrpm) execcmd(command, show=True) sys.exit(0)
def list_targets(option, opt, val, parser): host = config.get("submit", "host") if host is None: raise Error("no submit host defined in repsys.conf") createsrpm = get_helper("create-srpm") #TODO make it configurable args = ["ssh", host, createsrpm, "--list"] execcmd(args, show=true) sys.exit(0) # it is invoked via optparse callback, thus we need to
def default_parent(url): if url.find("://") == -1: default_parent = config.get("global", "default_parent") if not default_parent: raise Error, "received a relative url, " \ "but default_parent was not setup" parsed = list(urlparse.urlparse(default_parent)) parsed[2] = os.path.normpath(parsed[2] + "/" + url) url = urlparse.urlunparse(parsed) return url
def get_auth(username=None, password=None): set_username = 1 set_password = 1 if not username: username = config.get("auth", "username") if not username: username = raw_input("username: "******"auth", "password") if not password: password = getpass.getpass("password: "******"auth", "username", username) if set_password: config.set("auth", "password", password) return username, password
def download_binary(topdir, sha1, filename): fmt = config.get("global", "download-command", "wget -c -O '$dest' $url") url = config.get("binrepo", "download_url", "http://binrepo.mageia.org/") url = mirror.normalize_path(url + "/" + sha1) dest = os.path.join(topdir, 'SOURCES', filename) if os.path.exists(dest): if file_hash(dest) == sha1: return 1 else: raise Error("File with incorrect sha1sum: %s" % dest) context = {"dest": dest, "url": url} try: cmd = string.Template(fmt).substitute(context) except KeyError as e: raise Error("invalid variable %r in download-command "\ "configuration option" % e) try: status, output = execcmd(cmd, show=True) except Error as e: os.unlink(dest) raise Error("Could not download file %s\n" % url)
def filter_log_lines(lines): # Lines in commit messages beginning with CLOG will be the only shown # in the changelog. These lines will have the CLOG token and blanks # stripped from the beginning. onlylines = None clogstr = config.get("log", "unignore-string") if clogstr: clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr) onlylines = [clogre.sub("", line) for line in lines if line.startswith(clogstr)] if onlylines: filtered = onlylines else: # Lines in commit messages containing SILENT at any position will be # skipped; commits with their log messages beggining with SILENT in the # first positionj of the first line will have all lines ignored. ignstr = config.get("log", "ignore-string", "SILENT") if len(lines) and lines[0].startswith(ignstr): return [] filtered = [line for line in lines if ignstr not in line] return filtered
def _set_env(self): wrapper = "repsys-ssh" repsys = config.get("global", "repsys-cmd") if repsys: dir = os.path.dirname(repsys) path = os.path.join(dir, wrapper) if os.path.exists(path): wrapper = path defaults = {"SVN_SSH": wrapper} os.environ.update(defaults) raw = config.get("global", "svn-env") if raw: for line in raw.split("\n"): env = line.strip() if not env: continue try: name, value = env.split("=", 1) except ValueError: sys.stderr.write("invalid svn environment line: %r\n" % env) continue os.environ[name] = value
def filter_log_lines(lines): # Lines in commit messages beginning with CLOG will be the only shown # in the changelog. These lines will have the CLOG token and blanks # stripped from the beginning. onlylines = None clogstr = config.get("log", "unignore-string") if clogstr: clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr) onlylines = [ clogre.sub("", line) for line in lines if line.startswith(clogstr) ] if onlylines: filtered = onlylines else: # Lines in commit messages containing SILENT at any position will be # skipped; commits with their log messages beggining with SILENT in the # first positionj of the first line will have all lines ignored. ignstr = config.get("log", "ignore-string", "SILENT") if len(lines) and lines[0].startswith(ignstr): return [] filtered = [line for line in lines if ignstr not in line] return filtered
def expand_group(group): name, rev = layout.split_url_revision(group) distro = None if "/" in name: distro, name = name.rsplit("/", 1) found = config.get("submit-groups", name) packages = [group] if found: packages = found.split() if rev: packages = [("%s@%s" % (package, rev)) for package in packages] if distro: packages = ["%s/%s" % (distro, package) for package in packages] return packages
def delete(paths, commit=False): silent = config.get("log", "ignore-string", "SILENT") topdir = getpkgtopdir() svn = detectVCS(topdir) for path in paths: message = "%s: delete file %s" % (silent, path) if binrepo.is_binary(path): binrepo.update_sources(topdir, removed=[os.path.basename(path)]) if commit: svn.commit(binrepo.sources_path(topdir), log=message) else: svn.remove(path, local=True) if commit: svn.commit(path, log=message)
def mapurl(url): """Maps a url following the regexp provided by the option url-map in repsys.conf """ urlmap = config.get("global", "url-map") newurl = url if urlmap: try: expr_, replace = urlmap.split()[:2] except ValueError: log.error("invalid url-map: %s", urlmap) else: try: newurl = re.sub(expr_, replace, url) except re.error, errmsg: log.error("error in URL mapping regexp: %s", errmsg)
def _set_env(self): os.environ.update(self.env_defaults) if self.vcs_name: raw = config.get("global", "%s-env" % self.vcs_name) if raw: for line in raw.split("\n"): env = line.strip() if not env: continue try: name, value = env.split("=", 1) except ValueError: sys.stderr.write("invalid %s environment line: %r\n" % (self.vcs_name, env)) continue os.environ[name] = value
def build_rpm(build_cmd="a", verbose=False, rpmlint=True, packager = "", macros = []): top = os.getcwdu() topdir = "--define '_topdir %s'" % top builddir = "--define '_builddir %s/%s'" % (top, "BUILD") rpmdir = "--define '_rpmdir %s/%s'" % (top, "RPMS") sourcedir = "--define '_sourcedir %s/%s'" % (top, "SOURCES") specdir = "--define '_specdir %s/%s'" % (top, "SPECS") srcrpmdir = "--define '_srcrpmdir %s/%s'" % (top, "SRPMS") patchdir = "--define '_patchdir %s/%s'" % (top, "SOURCES") build = os.path.join(top, "BUILD") if not os.path.exists(build): os.mkdir(build) specsdir = os.path.join(top, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) if not speclist: raise Error, "no spec files found" spec = speclist[0] if packager: packager = " --define 'packager %s'" % packager defs = rpm_macros_defs(macros) rpmbuild = config.get("helper", "rpmbuild", "rpmbuild") begintime = time() execcmd("LC_ALL=C %s -b%s %s %s %s %s %s %s %s %s %s %s" % (rpmbuild, build_cmd, topdir, builddir, rpmdir, sourcedir, specdir, srcrpmdir, patchdir, packager, spec, defs), show=verbose) endtime = time() if rpmlint: status, output = execcmd("rpm %s %s --define 'debug_package_and_restore %%{debug_package}' --define '_query_all_fmt %%{_srcrpmdir}/%%{SOURCERPM} %%{_rpmdir}/%%{_build_name_fmt}' -q --specfile %s" % (rpmdir, srcrpmdir, spec)) checklist = [] for ps in output.split("\n"): for p in ps.split(): if p not in checklist and os.path.exists(p): # We assume that only packages with matching filename and # that has been built since we invoked rpmbuild belongs to # this build and should be checked stat = os.stat(p) if stat.st_ctime > begintime and stat.st_ctime < endtime: checklist.append(p) if checklist: execcmd("rpmlint %s" % string.join(checklist), show=True)
def dump_file(releases, currentlog=None, template=None): templpath = template or config.get("template", "path", "/usr/share/repsys/default.chlog") params = {} if templpath is None or not os.path.exists(templpath): params["source"] = default_template sys.stderr.write("warning: %s not found. using built-in template.\n"% templpath) else: params["file"] = templpath releases_author = group_releases_by_author(releases) revisions_author = group_revisions_by_author(currentlog) params["searchList"] = [{"releases_by_author" : releases_author, "releases" : releases, "revisions_by_author": revisions_author}] t = Template(**params) return t.respond()
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None): mirror.info(pkgdirurl) url = checkout_url(pkgdirurl) svncmd = config.get("global", "svn-command", "svn") args = [svncmd, "log", url] if verbose: args.append("-v") if limit: args.append("-l") args.append(limit) if revision: args.append("-r") args.append(revision) if os.isatty(sys.stdin.fileno()): args.append("| less") rawcmd = " ".join(args) execcmd(rawcmd, show=True)
def submit(urls, target, define=[], submithost=None, port=None, atonce=False, debug=False, keeplog=False): if submithost is None: submithost = config.get("submit", "host") if submithost is None: raise Error, "no submit host defined in configuration" if port is None: port = config.getint("submit", "port", "22") # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory createsrpm = get_helper("create-srpm") baseargs = ["ssh", "-p", str(port), submithost, createsrpm, "-t", target] if debug: baseargs.append("--debug") if keeplog: baseargs.append("--keeplog") for entry in reversed(define): baseargs.append("--define") baseargs.append(entry) cmdsargs = [] if len(urls) == 1: # be compatible with server-side repsys versions older than 1.6.90 url, rev = layout.split_url_revision(urls[0]) baseargs.append("-r") baseargs.append(str(rev)) baseargs.append(url) cmdsargs.append(baseargs) elif atonce: cmdsargs.append(baseargs + urls) else: cmdsargs.extend((baseargs + [url]) for url in urls) for cmdargs in cmdsargs: status, output = execcmd(cmdargs, noerror=1) if status == 0: print "Package submitted!" else: sys.stderr.write(output) sys.stderr.write("\n") sys.exit(status)
def users_wrapper(section, option=None, default=None, walk=False): global users_cache if walk: raise Error("ldapusers plugin does not support user listing") assert option is not None, \ "When not section walking, option is required" value = users_cache.get(option) if value is not None: return value try: #l = ldap.initialize(uri) l = Connection(uri, auto_bind=True, use_tls=starttls) # if starttls: # l.start_tls() #l.start_tls_s() if binddn: l.bind(binddn, bindpw) except ldap3.LDAPExceptionError as e: raise LDAPError(e) try: data = {"username": escape_filter_chars(option)} filter = interpolate("ldap-filterformat", filterformat, data) attrs = used_attributes(format) try: found = l.search(search_base='o='.basedn, search_filter=filter, attributes=attrs, search_scope=SUBTREE) # found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter, # attrlist=attrs) except ldap3.LDAPError as e: raise LDAPError(e) if found: dn, entry = found[0] entry = strip_entry(entry) value = interpolate("ldap-resultformat", format, entry) else: # issue a warning? value = config.get(section, option, default, wrap=False) users_cache[option] = value return value finally: l.unbind()
def _execsvn(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup") collecterr = False if kwargs.get("show"): if not kwargs.get("local"): collecterr = True else: if args[0] not in localcmds: args = list(args) args.append("--non-interactive") cleanerr = True self._set_env() svn_command = config.get("global", "svn-command", "svn") cmdargs = [svn_command] cmdargs.extend(args) try: return execcmd(cmdargs, show=kwargs.get("show", False), noerror=kwargs.get("noerror", False), collecterr=collecterr, cleanerr=cleanerr) except Error, e: msg = None if e.args: if "Permission denied" in e.args[0]: msg = ( "It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" " for more information.") elif "authorization failed" in e.args[0]: msg = ("Note that repsys does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error, "%s\n%s" % (e, msg) raise
def upload_binary(topdir, filename): filepath = os.path.join(topdir, 'SOURCES', filename) if not os.path.exists(filepath): raise Error("'%s' was not found" % filepath) sha1sum = file_hash(filepath) if binary_exists(sha1sum): return host = config.get("binrepo", "upload_host") upload_bin_helper = get_helper("upload-bin") command = ["ssh", host, upload_bin_helper, filename] try: filein = open(filepath, 'r') except Error as e: raise Error("Could not open file %s\n" % filepath) status, output = execcmd(*command, show=True, collecterr=True, stdin=filein)
def main(): if not os.environ.has_key('REQUEST_METHOD'): sys.stderr.write("error: this program is meant to be used as a cgi\n") sys.exit(1) print "Content-type: text/html\n\n" try: username = os.environ.get("REMOTE_USER") method = os.environ.get("REQUEST_METHOD") if not username or method != "POST": show() else: useremail = config.get("users", username) if not useremail: raise CgiError, \ "Your email was not found. Contact the administrator!" submit_packages(useremail) except CgiError, e: show(str(e), error=1)
def _execsvn(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup") collecterr = False if kwargs.get("show"): if not kwargs.get("local"): collecterr = True else: if args[0] not in localcmds: args = list(args) args.append("--non-interactive") cleanerr = True self._set_env() svn_command = config.get("global", "svn-command", "svn") cmdargs = [svn_command] cmdargs.extend(args) try: return execcmd(cmdargs, show=kwargs.get("show", False), noerror=kwargs.get("noerror", False), collecterr=collecterr, cleanerr=cleanerr) except Error, e: msg = None if e.args: if "Permission denied" in e.args[0]: msg = ("It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration" " for more information.") elif "authorization failed" in e.args[0]: msg = ("Note that repsys does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error, "%s\n%s" % (e, msg) raise
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None): mirror.info(pkgdirurl) url = checkout_url(pkgdirurl) svncmd = config.get("global", "svn-command", "svn") args = [svncmd, "log", url] if verbose: args.append("-v") if limit: args.append("-l") args.append(str(limit)) if revision: args.append("-r") args.append(revision) if os.isatty(sys.stdin.fileno()): pager = shlex.split(os.environ.get("PAGER", "less")) p = subprocess.Popen(args, stdout=subprocess.PIPE) p2 = subprocess.Popen(pager, stdin=p.stdout) p2.wait() p.wait() else: execcmd(args, show=True)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): if submithost is None: submithost = config.get("submit", "host") if submithost is None: # extract the submit host from the svn host type, rest = urllib.splittype(pkgdirurl) host, path = urllib.splithost(rest) user, host = urllib.splituser(host) submithost, port = urllib.splitport(host) del type, user, port, path, rest # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory createsrpm = get_helper("create-srpm") baseargs = ["ssh", submithost, createsrpm, "-t", target] if not sid: sid = uuid.uuid4() define.append("sid=%s" % sid) for entry in reversed(define): baseargs.append("--define") baseargs.append(entry) cmdsargs = [] if len(urls) == 1: # be compatible with server-side repsys versions older than 1.6.90 url, rev = layout.split_url_revision(urls[0]) baseargs.append("-r") baseargs.append(str(rev)) baseargs.append(url) cmdsargs.append(baseargs) elif atonce: cmdsargs.append(baseargs + urls) else: cmdsargs.extend((baseargs + [url]) for url in urls) for cmdargs in cmdsargs: command = subprocess.list2cmdline(cmdargs) status, output = execcmd(command) if status == 0: print "Package submitted!" else: sys.stderr.write(output) sys.exit(status)
def svn2rpm(pkgdirurl, rev=None, size=None, submit=False, template=None, macros=[], exported=None): concat = config.get("log", "concat", "").split() revoffset = get_revision_offset() svn = SVN() pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True) pkgcurrenturl = layout.checkout_url(pkgdirurl) releaseslog = svn.log(pkgreleasesurl, noerror=1) currentlog = svn.log(pkgcurrenturl, limit=size, start=rev, end=revoffset) # sort releases by copyfrom-revision, so that markreleases for same # revisions won't look empty releasesdata = [] if releaseslog: for relentry in releaseslog[::-1]: try: (version, release, relrevision) = \ parse_markrelease_log(relentry) except InvalidEntryError: continue releasesdata.append((relrevision, -relentry.revision, relentry, version, release)) releasesdata.sort() # collect valid releases using the versions provided by the changes and # the packages prevrevision = 0 releases = [] for (relrevision, dummy, relentry, version, release) in releasesdata: if prevrevision == relrevision: # ignore older markrelease of the same revision, since they # will have no history continue entries = [entry for entry in currentlog if relrevision >= entry.revision and (prevrevision < entry.revision)] if not entries: #XXX probably a forced release, without commits in current/, # check if this is the right behavior sys.stderr.write("warning: skipping (possible) release " "%s-%s@%s, no commits since previous markrelease (r%r)\n" % (version, release, relrevision, prevrevision)) continue release = make_release(author=relentry.author, revision=relentry.revision, date=relentry.date, lines=relentry.lines, entries=entries, version=version, release=release) releases.append(release) prevrevision = relrevision # look for commits that have been not submitted (released) yet # this is done by getting all log entries newer (greater revision no.) # than releasesdata[-1] (in the case it exists) if releasesdata: latest_revision = releasesdata[-1][0] # the latest copied rev else: latest_revision = 0 notsubmitted = [entry for entry in currentlog if entry.revision > latest_revision] if notsubmitted: # if they are not submitted yet, what we have to do is to add # a release/version number from getrelease() version, release = getrelease(pkgdirurl, macros=macros, exported=exported) toprelease = make_release(entries=notsubmitted, released=False, version=version, release=release) releases.append(toprelease) data = dump_file(releases[::-1], currentlog=currentlog, template=template) return data
def __init__(self): VCS.__init__(self) self.vcs_name = "git" self.vcs_command = config.get("global", "git-command", "git") self.vcs_supports['clone'] = True self.env_defaults = {"GIT_SSH": self.vcs_wrapper}
def author_email(self, author): return config.get("users", author)
def mirror_url(): mirror = config.get("global", "mirror") return mirror
def layout_dirs(): devel_branch = config.get("global", "trunk-dir", "cooker/") devel_branch = os.path.normpath(devel_branch) branches_dir = config.get("global", "branches-dir", "updates/") branches_dir = os.path.normpath(branches_dir) return devel_branch, branches_dir
def get_author_name(author): found = emailpat.match(config.get("users", author, author)) name = ((found and found.group("name")) or author) email = ((found and found.group("email")) or author) return name, email
def get_srpm(pkgdirurl, mode = "current", targetdirs = None, version = None, release = None, revision = None, packager = "", revname = 0, svnlog = 0, scripts = [], submit = False, template = None, distro = None, macros = [], verbose = 0, strict = False): svn = SVN() tmpdir = tempfile.mktemp() topdir = "_topdir %s" % tmpdir builddir = "_builddir %s/%s" % (tmpdir, "BUILD") rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS") sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES") specdir = "_specdir %s/%s" % (tmpdir, "SPECS") srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS") patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES") temppath = "_tmppath %s" % (tmpdir) rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir, sourcedir, specdir, srcrpmdir, patchdir, temppath)] try: if mode == "version": geturl = layout.checkout_url(pkgdirurl, version=version, release=release) elif mode == "pristine": geturl = layout.checkout_url(pkgdirurl, pristine=True) elif mode == "current" or mode == "revision": #FIXME we should handle revisions specified using @REV geturl = layout.checkout_url(pkgdirurl) else: raise Error, "unsupported get_srpm mode: %s" % mode strict = strict or config.getbool("submit", "strict-revision", False) if strict and not rev_touched_url(geturl, revision): #FIXME would be nice to have the revision number even when # revision is None raise Error, "the revision %s does not change anything "\ "inside %s" % (revision or "HEAD", geturl) mirror.info(geturl) svn.export(geturl, tmpdir, rev=revision) srpmsdir = os.path.join(tmpdir, "SRPMS") os.mkdir(srpmsdir) specsdir = os.path.join(tmpdir, "SPECS") speclist = glob.glob(os.path.join(specsdir, "*.spec")) if config.getbool("srpm", "run-prep", False): makefile = os.path.join(tmpdir, "Makefile") if os.path.exists(makefile): execcmd(("make", "-C", tmpdir, "srpm-prep")) if not speclist: raise Error, "no spec files found" spec = speclist[0] if svnlog: submit = not not revision log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit, template=template, macros=macros, exported=tmpdir) for script in scripts: #FIXME revision can be "None" status, output = execcmd(script, tmpdir, spec, str(revision), noerror=1) if status != 0: raise Error, "script %s failed" % script if packager: packager = " --define 'packager %s'" % packager if distro: cmpdistro = distro.lower() for target in cgiutil.get_targets(): if target.name.lower() == cmpdistro: macros.extend(target.macros) break else: raise Error, "no such submit target in configuration: %s" % (distro) sourcecmd = config.get("helper", "rpmbuild", "rpmbuild") args = [sourcecmd, "-bs", "--nodeps"] for pair in rpmdefs: args.extend(pair) for pair in macros: args.extend(("--define", "%s %s" % pair)) args.append(spec) try: execcmd(args) except CommandError, e: if config.getbool("global", "verbose"): cmdline = e.cmdline + "\n" else: cmdline = "" raise Error, ("error while creating the source RPM " "(with %s):\n%s%s" % (sourcecmd, cmdline, e.output)) # copy the generated SRPMs to their target locations targetsrpms = [] urlrev = None if revname: urlrev = revision or layout.get_url_revision(geturl) if not targetdirs: targetdirs = (".",) srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm")) if not srpms: # something fishy happened raise Error, "no SRPMS were found at %s" % srpmsdir for srpm in srpms: name = os.path.basename(srpm) if revname: name = "@%s:%s" % (urlrev, name) for targetdir in targetdirs: newpath = os.path.join(targetdir, name) targetsrpms.append(newpath) if os.path.exists(newpath): # should we warn? os.unlink(newpath) shutil.copy(srpm, newpath) if verbose: sys.stderr.write("Wrote: %s\n" % newpath) return targetsrpms
def dummy_wrapper(section, option=None, default=None, walk=False): return config.get(section, option, default, wrap=False)
def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None, baseurl=None, baseold=None, logmsg=None, rename=True): svn = SVN() srpm = SRPM(srpmfile) tmpdir = tempfile.mktemp() if baseurl: pkgurl = mirror._joinurl(baseurl, srpm.name) else: pkgurl = layout.package_url(srpm.name, distro=branch, mirrored=False) print "Importing package to %s" % pkgurl try: if srpm.epoch: version = "%s:%s" % (srpm.epoch, srpm.version) else: version = srpm.version versionurl = "/".join([pkgurl, "releases", version]) releaseurl = "/".join([versionurl, srpm.release]) currenturl = "/".join([pkgurl, "current"]) currentdir = os.path.join(tmpdir, "current") #FIXME when pre-commit hook fails, there's no clear way to know # what happened ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory") if ret or not svn.ls(currenturl, noerror=1): svn.checkout(pkgurl, tmpdir) svn.mkdir(os.path.join(tmpdir, "releases")) svn.mkdir(currentdir) svn.mkdir(os.path.join(currentdir, "SPECS")) svn.mkdir(os.path.join(currentdir, "SOURCES")) #svn.commit(tmpdir,log="Created package structure.") version_exists = 1 else: if svn.ls(releaseurl, noerror=1): raise Error, "release already exists" svn.checkout("/".join([pkgurl, "current"]), tmpdir) svn.mkdir(versionurl, noerror=1, log="Created directory for version %s." % version) currentdir = tmpdir specsdir = os.path.join(currentdir, "SPECS") sourcesdir = os.path.join(currentdir, "SOURCES") unpackdir = tempfile.mktemp() os.mkdir(unpackdir) try: srpm.unpack(unpackdir) uspecsdir = os.path.join(unpackdir, "SPECS") usourcesdir = os.path.join(unpackdir, "SOURCES") uspecsentries = os.listdir(uspecsdir) usourcesentries = os.listdir(usourcesdir) specsentries = os.listdir(specsdir) sourcesentries = os.listdir(sourcesdir) # Remove old entries for entry in [x for x in specsentries if x not in uspecsentries]: if entry == ".svn": continue entrypath = os.path.join(specsdir, entry) os.unlink(entrypath) svn.remove(entrypath) for entry in [x for x in sourcesentries if x not in usourcesentries]: if entry == ".svn": continue entrypath = os.path.join(sourcesdir, entry) os.unlink(entrypath) svn.remove(entrypath) # Copy all files execcmd(["cp", "-rf", uspecsdir, currentdir]) execcmd(["cp", "-rf", usourcesdir, currentdir]) # Add new entries for entry in [x for x in uspecsentries if x not in specsentries]: entrypath = os.path.join(specsdir, entry) svn.add(entrypath) for entry in [x for x in usourcesentries if x not in sourcesentries]: entrypath = os.path.join(sourcesdir, entry) svn.add(entrypath) finally: if os.path.isdir(unpackdir): shutil.rmtree(unpackdir) specs = glob.glob(os.path.join(specsdir, "*.spec")) if not specs: raise Error, "no spec file found on %s" % specsdir if len(specs) > 1: raise Error, "more than one spec file found on %s" % specsdir specpath = specs[0] if rename: specfile = os.path.basename(specpath) specname = specfile[:-len(".spec")] if specname != srpm.name: newname = srpm.name + ".spec" newpath = os.path.join(specsdir, newname) sys.stderr.write("warning: renaming spec file to '%s' " "(use -n to disable it)\n" % (newname)) os.rename(specpath, newpath) try: svn.remove(specpath) except Error: # file not tracked svn.revert(specpath) svn.add(newpath) specpath = newpath if striplog: specpath = specpath fspec = open(specpath) spec, chlog = log.split_spec_changelog(fspec) fspec.close() fspec = open(specpath, "w") fspec.writelines(spec) fspec.close() chlog.seek(0, os.SEEK_END) if chlog.tell() != 0: chlog.seek(0) #FIXME move it to layout.py oldurl = baseold or config.get("log", "oldurl") pkgoldurl = mirror._joinurl(oldurl, srpm.name) svn.mkdir(pkgoldurl, noerror=1, log="created old log directory for %s" % srpm.name) logtmp = tempfile.mktemp() try: svn.checkout(pkgoldurl, logtmp) miscpath = os.path.join(logtmp, "log") fmisc = open(miscpath, "w+") fmisc.writelines(chlog) fmisc.close() svn.add(miscpath) svn.commit(logtmp, log="imported old log for %s" % srpm.name) finally: if os.path.isdir(logtmp): shutil.rmtree(logtmp) svn.commit(tmpdir, log=logmsg or ("imported package %s" % srpm.name)) finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) # Do revision and pristine tag copies pristineurl = layout.checkout_url(pkgurl, pristine=True) svn.remove(pristineurl, noerror=1, log="Removing previous pristine/ directory.") currenturl = layout.checkout_url(pkgurl) svn.copy(currenturl, pristineurl, log="Copying release %s-%s to pristine/ directory." % (version, srpm.release)) if markrelease: svn.copy(currenturl, releaseurl, log="Copying release %s-%s to releases/ directory." % (version, srpm.release))
if specstl: specst, _ = specstl[0] if specst == "?": toadd.append(specpath) # add source files: for source, url in sources.iteritems(): sourcepath = os.path.join(sourcesdir, source) pst = sourcesst.get(source) if pst: if os.path.isfile(sourcepath): toadd.append(sourcepath) else: sys.stderr.write("warning: %s not found, skipping\n" % sourcepath) elif download and not os.path.isfile(sourcepath): print "%s not found, downloading from %s" % (sourcepath, url) fmt = config.get("global", "download-command", "wget -c -O '$dest' $url") context = {"dest": sourcepath, "url": url} try: cmd = string.Template(fmt).substitute(context) except KeyError, e: raise Error, "invalid variable %r in download-command "\ "configuration option" % e execcmd(cmd, show=True) if os.path.isfile(sourcepath): toadd.append(sourcepath) else: raise Error, "file not found: %s" % sourcepath # rm entries not found in sources and still in svn found = os.listdir(sourcesdir) toremove = [] for entry in found: