def get_helper(name): """Tries to find the path of a helper script It first looks if the helper has been explicitly defined in configuration, if not, falls back to the default helper path, which can also be defined in configuration file(s). """ helperdir = config.get("helper", "prefix", "/usr/share/mgarepo") hpath = config.get("helper", name, None) or \ os.path.join(helperdir, name) return hpath
def repository_url(mirrored=False): url = None if mirrored and config.getbool("global", "use-mirror", "yes"): url = config.get("global", "mirror") if url is None: url = config.get("global", "repository") if not url: # compatibility with the default_parent configuration option default_parent = config.get("global", "default_parent") if default_parent is None: raise Error, "you need to set the 'repository' " \ "configuration option on mgarepo.conf" url = convert_default_parent(default_parent) return url
def binary_exists(sha1sum): dlurl = config.get("binrepo", "download_url", "http://binrepo.mageia.org/") dlurl = mirror.normalize_path(dlurl + "/" + sha1sum) h = httplib2.Http() resp, content = h.request(dlurl, 'HEAD') return resp.status == 200
def submit_package(self, packageurl, packagerev, targetname): username = os.environ.get("REMOTE_USER") packager = config.get("users", username) if not packager: raise CgiError, "your email was not found" elif not packagerev: raise CgiError, "no revision provided" elif not targetname: raise CgiError, "no target provided" else: targetname = targetname.lower() for target in get_targets(): if target.name.lower() == targetname: break else: raise CgiError, "target not found" try: tmp = int(packagerev) except ValueError: raise CgiError, "invalid revision provided" for allowed in target.allowed: if packageurl.startswith(allowed): break else: raise CgiError, "%s is not allowed for this target" \ % packageurl get_srpm(packageurl, revision=packagerev, targetdirs=target.target, packager=packager, revname=1, svnlog=1, scripts=target.scripts) return 1
def maintdb(maintdb_args): host = config.get("maintdb", "host", "maintdb.mageia.org") maintdb_helper = get_helper("maintdb") cmd_args = ' '.join(maintdb_args) command = "ssh %s %s %s" % (host, maintdb_helper, cmd_args) execcmd(command, show=True) sys.exit(0)
def get_old_log(pkgdirurl): chlog = StringIO() oldurl = config.get("log", "oldurl") if oldurl: svn = SVN() tmpdir = tempfile.mktemp() try: if oldurl == '.' or oldurl.startswith('./'): pkgoldurl = os.path.join(pkgdirurl, oldurl) else: pkgname = layout.package_name(pkgdirurl) pkgoldurl = os.path.join(oldurl, pkgname) try: # we're using HEAD here because fixes in misc/ (oldurl) may # be newer than packages' last changed revision. svn.export(pkgoldurl, tmpdir) except Error: pass else: logfile = os.path.join(tmpdir, "log") if os.path.isfile(logfile): file = open(logfile) chlog.write("\n") # TODO needed? log = file.read() log = escape_macros(log) chlog.write(log) file.close() finally: if os.path.isdir(tmpdir): shutil.rmtree(tmpdir) chlog.seek(0) return chlog
def upload(paths, commit=False): for path in paths: if os.path.isdir(path) or binrepo.is_binary(path): topdir = getpkgtopdir() binrepo.upload_binary(topdir, os.path.basename(path)) binrepo.update_sources(topdir, added=[path]) if commit: svn = SVN() silent = config.get("log", "ignore-string", "SILENT") message = "%s: new file %s" % (silent, path) svn.commit(binrepo.sources_path(topdir), log=message) else: svn = SVN() svn.add(path, local=True) if commit: silent = config.get("log", "ignore-string", "SILENT") message = "%s: new file %s" % (silent, path) svn.commit(path, log=message)
def maintdb_get(package): dlurl = config.get("maintdb", "url", "http://maintdb.mageia.org/") dlurl = dlurl + "/" + package h = httplib2.Http() resp, content = h.request(dlurl, 'GET') if resp.status != 200: raise Exception('Package cannot be found in maintdb') return content.rstrip('\n')
def download_binary(topdir, sha1, filename): fmt = config.get("global", "download-command", "wget -c -O '$dest' $url") url = config.get("binrepo", "download_url", "http://binrepo.mageia.org/") url = mirror.normalize_path(url + "/" + sha1) dest = os.path.join(topdir, 'SOURCES', filename) if os.path.exists(dest): if file_hash(dest) == sha1: return 1 else: raise Error, "File with incorrect sha1sum: %s" % dest context = {"dest": dest, "url": url} try: cmd = string.Template(fmt).substitute(context) except KeyError, e: raise Error, "invalid variable %r in download-command "\ "configuration option" % e
def list_targets(option, opt, val, parser): host = config.get("submit", "host") if host is None: raise Error, "no submit host defined in mgarepo.conf" createsrpm = get_helper("create-srpm") #TODO make it configurable command = "ssh %s %s --list" % (host, createsrpm) execcmd(command, show=True) sys.exit(0)
def default_parent(url): if url.find("://") == -1: default_parent = config.get("global", "default_parent") if not default_parent: raise Error, "received a relative url, " \ "but default_parent was not setup" parsed = list(urlparse.urlparse(default_parent)) parsed[2] = os.path.normpath(parsed[2] + "/" + url) url = urlparse.urlunparse(parsed) return url
def get_auth(username=None, password=None): set_username = 1 set_password = 1 if not username: username = config.get("auth", "username") if not username: username = raw_input("username: "******"auth", "password") if not password: password = getpass.getpass("password: "******"auth", "username", username) if set_password: config.set("auth", "password", password) return username, password
def filter_log_lines(lines): # Lines in commit messages beginning with CLOG will be the only shown # in the changelog. These lines will have the CLOG token and blanks # stripped from the beginning. onlylines = None clogstr = config.get("log", "unignore-string") if clogstr: clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr) onlylines = [clogre.sub("", line) for line in lines if line.startswith(clogstr)] if onlylines: filtered = onlylines else: # Lines in commit messages containing SILENT at any position will be # skipped; commits with their log messages beggining with SILENT in the # first positionj of the first line will have all lines ignored. ignstr = config.get("log", "ignore-string", "SILENT") if len(lines) and lines[0].startswith(ignstr): return [] filtered = [line for line in lines if ignstr not in line] return filtered
def _set_env(self): wrapper = "mgarepo-ssh" repsys = config.get("global", "mgarepo-cmd") if repsys: dir = os.path.dirname(repsys) path = os.path.join(dir, wrapper) if os.path.exists(path): wrapper = path defaults = {"SVN_SSH": wrapper} os.environ.update(defaults) raw = config.get("global", "svn-env") if raw: for line in raw.split("\n"): env = line.strip() if not env: continue try: name, value = env.split("=", 1) except ValueError: sys.stderr.write("invalid svn environment line: %r\n" % env) continue os.environ[name] = value
def upload_binary(topdir, filename): filepath = os.path.join(topdir, 'SOURCES', filename) if not os.path.exists(filepath): raise Error, "'%s' was not found" % filepath sha1sum = file_hash(filepath) if binary_exists(sha1sum): return host = config.get("binrepo", "upload_host") upload_bin_helper = get_helper("upload-bin") command = "ssh %s %s %s" % (host, upload_bin_helper, filename) try: filein = open(filepath, 'r') except Error, e: raise Error, "Could not open file %s\n" % filepath
def is_binary(path): raw = config.get("binrepo", "upload-match", "\.(7z|Z|bin|bz2|cpio|db|deb|egg|gem|gz|jar|jisp|lzma|"\ "pdf|pgn\\.gz|pk3|png|rpm|run|sdz|smzip|tar|tbz|"\ "tbz2|tgz|ttf|uqm|wad|war|xar|xpi|xz|zip|wav|mp3|ogg|"\ "jpg|png|gif|avi|mpg|mpeg|rar)$") maxsize = config.getint("binrepo", "upload-match-size", "1048576") # 1MiB expr = re.compile(raw) name = os.path.basename(path) if expr.search(name): return True st = os.stat(path) if st[stat.ST_SIZE] >= maxsize: return True return False
def delete(paths, commit=False): silent = config.get("log", "ignore-string", "SILENT") for path in paths: message = "%s: delete file %s" % (silent, path) if binrepo.is_binary(path): topdir = getpkgtopdir() binrepo.update_sources(topdir, removed=[os.path.basename(path)]) if commit: svn = SVN() svn.commit(binrepo.sources_path(topdir), log=message) else: svn = SVN() svn.remove(path, local=True) if commit: svn.commit(path, log=message)
def expand_group(group): name, rev = layout.split_url_revision(group) distro = None if "/" in name: distro, name = name.rsplit("/", 1) found = config.get("submit-groups", name) packages = [group] if found: packages = found.split() if rev: packages = [("%s@%s" % (package, rev)) for package in packages] if distro: packages = ["%s/%s" % (distro, package) for package in packages] return packages
def mapurl(url): """Maps a url following the regexp provided by the option url-map in mgarepo.conf """ urlmap = config.get("global", "url-map") newurl = url if urlmap: try: expr_, replace = urlmap.split()[:2] except ValueError: sys.stderr.write("invalid url-map: %s" % urlmap) else: try: newurl = re.sub(expr_, replace, url) except re.error, errmsg: sys.stderr.write("error in URL mapping regexp: %s" % errmsg)
def dump_file(releases, currentlog=None, template=None): templpath = template or config.get("template", "path", "/usr/share/mgarepo/default.chlog") params = {} if templpath is None or not os.path.exists(templpath): params["source"] = default_template sys.stderr.write("warning: %s not found. using built-in template.\n"% templpath) else: params["file"] = templpath releases_author = group_releases_by_author(releases) revisions_author = group_revisions_by_author(currentlog) params["searchList"] = [{"releases_by_author" : releases_author, "releases" : releases, "revisions_by_author": revisions_author}] t = Template(**params) return t.respond()
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None): mirror.info(pkgdirurl) url = checkout_url(pkgdirurl) svncmd = config.get("global", "svn-command", "svn") args = [svncmd, "log", url] if verbose: args.append("-v") if limit: args.append("-l") args.append(limit) if revision: args.append("-r") args.append(revision) if os.isatty(sys.stdin.fileno()): args.append("| less") rawcmd = " ".join(args) execcmd(rawcmd, show=True)
def main(): if not os.environ.has_key('REQUEST_METHOD'): sys.stderr.write("error: this program is meant to be used as a cgi\n") sys.exit(1) print "Content-type: text/html\n\n" try: username = os.environ.get("REMOTE_USER") method = os.environ.get("REQUEST_METHOD") if not username or method != "POST": show() else: useremail = config.get("users", username) if not useremail: raise CgiError, \ "Your email was not found. Contact the administrator!" submit_packages(useremail) except CgiError, e: show(str(e), error=1)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None): if submithost is None: submithost = config.get("submit", "host") if submithost is None: # extract the submit host from the svn host type, rest = urllib.splittype(pkgdirurl) host, path = urllib.splithost(rest) user, host = urllib.splituser(host) submithost, port = urllib.splitport(host) del type, user, port, path, rest # runs a create-srpm in the server through ssh, which will make a # copy of the rpm in the export directory createsrpm = get_helper("create-srpm") baseargs = ["ssh", submithost, createsrpm, "-t", target] if not sid: sid = uuid.uuid4() define.append("sid=%s" % sid) for entry in reversed(define): baseargs.append("--define") baseargs.append(entry) cmdsargs = [] if len(urls) == 1: # be compatible with server-side mgarepo versions older than 1.6.90 url, rev = layout.split_url_revision(urls[0]) baseargs.append("-r") baseargs.append(str(rev)) baseargs.append(url) cmdsargs.append(baseargs) elif atonce: cmdsargs.append(baseargs + urls) else: cmdsargs.extend((baseargs + [url]) for url in urls) for cmdargs in cmdsargs: command = subprocess.list2cmdline(cmdargs) status, output = execcmd(command) if status == 0: print "Package submitted!" else: sys.stderr.write(output) sys.exit(status)
def _execsvn(self, *args, **kwargs): localcmds = ("add", "revert", "cleanup") if not kwargs.get("show") and args[0] not in localcmds: args = list(args) args.append("--non-interactive") else: kwargs["geterr"] = True kwargs["cleanerr"] = True if kwargs.get("xml"): args.append("--xml") self._set_env() svn_command = config.get("global", "svn-command", "svn") cmdstr = svn_command + " " + " ".join(args) try: return execcmd(cmdstr, **kwargs) except Error, e: msg = None if e.args: if "Permission denied" in e.args[0]: msg = ("It seems ssh-agent or ForwardAgent are not setup " "or your username is wrong. See " "https://wiki.mageia.org/en/Packagers_ssh" " for more information.") elif "authorization failed" in e.args[0]: msg = ("Note that mgarepo does not support any HTTP " "authenticated access.") if kwargs.get("show") and \ not config.getbool("global", "verbose", 0): # svn has already dumped error messages, we don't need to # do it too if msg: sys.stderr.write("\n") sys.stderr.write(msg) sys.stderr.write("\n") raise SilentError elif msg: raise Error, "%s\n%s" % (e, msg) raise
def dummy_wrapper(section, option=None, default=None, walk=False): return config.get(section, option, default, wrap=False)
def make_handler(): uri = config.get("global", "ldap-uri") if not uri: server = config.get("global", "ldap-server") if not server: # ldap support is not enabled if ldap-uri nor ldap-server are # defined def dummy_wrapper(section, option=None, default=None, walk=False): return config.get(section, option, default, wrap=False) return dummy_wrapper try: port = int(config.get("global", "ldap-port", 389)) except ValueError: raise Error, "the option ldap-port requires an integer, please "\ "check your configuration files" uri = "ldap://%s:%d" % (server, port) basedn = config.get("global", "ldap-base") binddn = config.get("global", "ldap-binddn") bindpw = config.get("global", "ldap-bindpw", "") filterformat = config.get("global", "ldap-filterformat", "(&(objectClass=inetOrgPerson)(uid=$username))", raw=1) format = config.get("global", "ldap-resultformat", "$cn <$mail>", raw=1) valid = {"yes": True, "no": False} raw = config.get("global", "ldap-starttls", "no") try: starttls = valid[raw] except KeyError: raise Error, "invalid value %r for ldap-starttls, use "\ "'yes' or 'no'" % raw try: import ldap except ImportError: raise Error, "LDAP support needs the python-ldap package "\ "to be installed" else: from ldap.filter import escape_filter_chars def users_wrapper(section, option=None, default=None, walk=False): global users_cache if walk: raise Error, "ldapusers plugin does not support user listing" assert option is not None, \ "When not section walking, option is required" value = users_cache.get(option) if value is not None: return value try: l = ldap.initialize(uri) if starttls: l.start_tls_s() if binddn: l.bind(binddn, bindpw) except ldap.LDAPError, e: raise LDAPError(e) try: data = {"username": escape_filter_chars(option)} filter = interpolate("ldap-filterformat", filterformat, data) attrs = used_attributes(format) try: found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter, attrlist=attrs) except ldap.LDAPError, e: raise LDAPError(e) if found: dn, entry = found[0] entry = strip_entry(entry) value = interpolate("ldap-resultformat", format, entry) else: # issue a warning? value = config.get(section, option, default, wrap=False) users_cache[option] = value return value
def mirror_url(): mirror = config.get("global", "mirror") return mirror
def layout_dirs(): devel_branch = config.get("global", "trunk-dir", "cauldron/") devel_branch = os.path.normpath(devel_branch) branches_dir = config.get("global", "branches-dir", "updates/") branches_dir = os.path.normpath(branches_dir) return devel_branch, branches_dir
def author_email(self, author): return config.get("users", author)
def print_author_email(author): email = config.get("users", author) if not email: raise Error, "author not found" print email