def get_helper(name):
"""Tries to find the path of a helper script
It first looks if the helper has been explicitly defined in
configuration, if not, falls back to the default helper path, which can
also be defined in configuration file(s).
"""
helperdir = config.get("helper", "prefix", "/usr/share/mgarepo")
hpath = config.get("helper", name, None) or \
os.path.join(helperdir, name)
return hpath
def repository_url(mirrored=False):
url = None
if mirrored and config.getbool("global", "use-mirror", "yes"):
url = config.get("global", "mirror")
if url is None:
url = config.get("global", "repository")
if not url:
# compatibility with the default_parent configuration option
default_parent = config.get("global", "default_parent")
if default_parent is None:
raise Error, "you need to set the 'repository' " \
"configuration option on mgarepo.conf"
url = convert_default_parent(default_parent)
return url
def binary_exists(sha1sum):
dlurl = config.get("binrepo", "download_url",
"http://binrepo.mageia.org/")
dlurl = mirror.normalize_path(dlurl + "/" + sha1sum)
h = httplib2.Http()
resp, content = h.request(dlurl, 'HEAD')
return resp.status == 200
def submit_package(self, packageurl, packagerev, targetname):
username = os.environ.get("REMOTE_USER")
packager = config.get("users", username)
if not packager:
raise CgiError, "your email was not found"
elif not packagerev:
raise CgiError, "no revision provided"
elif not targetname:
raise CgiError, "no target provided"
else:
targetname = targetname.lower()
for target in get_targets():
if target.name.lower() == targetname:
break
else:
raise CgiError, "target not found"
try:
tmp = int(packagerev)
except ValueError:
raise CgiError, "invalid revision provided"
for allowed in target.allowed:
if packageurl.startswith(allowed):
break
else:
raise CgiError, "%s is not allowed for this target" \
% packageurl
get_srpm(packageurl,
revision=packagerev,
targetdirs=target.target,
packager=packager,
revname=1,
svnlog=1,
scripts=target.scripts)
return 1
def maintdb(maintdb_args):
host = config.get("maintdb", "host", "maintdb.mageia.org")
maintdb_helper = get_helper("maintdb")
cmd_args = ' '.join(maintdb_args)
command = "ssh %s %s %s" % (host, maintdb_helper, cmd_args)
execcmd(command, show=True)
sys.exit(0)
def get_old_log(pkgdirurl):
chlog = StringIO()
oldurl = config.get("log", "oldurl")
if oldurl:
svn = SVN()
tmpdir = tempfile.mktemp()
try:
if oldurl == '.' or oldurl.startswith('./'):
pkgoldurl = os.path.join(pkgdirurl, oldurl)
else:
pkgname = layout.package_name(pkgdirurl)
pkgoldurl = os.path.join(oldurl, pkgname)
try:
# we're using HEAD here because fixes in misc/ (oldurl) may
# be newer than packages' last changed revision.
svn.export(pkgoldurl, tmpdir)
except Error:
pass
else:
logfile = os.path.join(tmpdir, "log")
if os.path.isfile(logfile):
file = open(logfile)
chlog.write("\n") # TODO needed?
log = file.read()
log = escape_macros(log)
chlog.write(log)
file.close()
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
chlog.seek(0)
return chlog
def upload(paths, commit=False):
for path in paths:
if os.path.isdir(path) or binrepo.is_binary(path):
topdir = getpkgtopdir()
binrepo.upload_binary(topdir, os.path.basename(path))
binrepo.update_sources(topdir, added=[path])
if commit:
svn = SVN()
silent = config.get("log", "ignore-string", "SILENT")
message = "%s: new file %s" % (silent, path)
svn.commit(binrepo.sources_path(topdir), log=message)
else:
svn = SVN()
svn.add(path, local=True)
if commit:
silent = config.get("log", "ignore-string", "SILENT")
message = "%s: new file %s" % (silent, path)
svn.commit(path, log=message)
def maintdb_get(package):
dlurl = config.get("maintdb", "url",
"http://maintdb.mageia.org/")
dlurl = dlurl + "/" + package
h = httplib2.Http()
resp, content = h.request(dlurl, 'GET')
if resp.status != 200:
raise Exception('Package cannot be found in maintdb')
return content.rstrip('\n')
def download_binary(topdir, sha1, filename):
fmt = config.get("global", "download-command",
"wget -c -O '$dest' $url")
url = config.get("binrepo", "download_url",
"http://binrepo.mageia.org/")
url = mirror.normalize_path(url + "/" + sha1)
dest = os.path.join(topdir, 'SOURCES', filename)
if os.path.exists(dest):
if file_hash(dest) == sha1:
return 1
else:
raise Error, "File with incorrect sha1sum: %s" % dest
context = {"dest": dest, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError, e:
raise Error, "invalid variable %r in download-command "\
"configuration option" % e
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in mgarepo.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
command = "ssh %s %s --list" % (host, createsrpm)
execcmd(command, show=True)
sys.exit(0)
def default_parent(url):
if url.find("://") == -1:
default_parent = config.get("global", "default_parent")
if not default_parent:
raise Error, "received a relative url, " \
"but default_parent was not setup"
parsed = list(urlparse.urlparse(default_parent))
parsed[2] = os.path.normpath(parsed[2] + "/" + url)
url = urlparse.urlunparse(parsed)
return url
def get_auth(username=None, password=None):
set_username = 1
set_password = 1
if not username:
username = config.get("auth", "username")
if not username:
username = raw_input("username: "******"auth", "password")
if not password:
password = getpass.getpass("password: "******"auth", "username", username)
if set_password:
config.set("auth", "password", password)
return username, password
def filter_log_lines(lines):
# Lines in commit messages beginning with CLOG will be the only shown
# in the changelog. These lines will have the CLOG token and blanks
# stripped from the beginning.
onlylines = None
clogstr = config.get("log", "unignore-string")
if clogstr:
clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr)
onlylines = [clogre.sub("", line)
for line in lines if line.startswith(clogstr)]
if onlylines:
filtered = onlylines
else:
# Lines in commit messages containing SILENT at any position will be
# skipped; commits with their log messages beggining with SILENT in the
# first positionj of the first line will have all lines ignored.
ignstr = config.get("log", "ignore-string", "SILENT")
if len(lines) and lines[0].startswith(ignstr):
return []
filtered = [line for line in lines if ignstr not in line]
return filtered
def _set_env(self):
wrapper = "mgarepo-ssh"
repsys = config.get("global", "mgarepo-cmd")
if repsys:
dir = os.path.dirname(repsys)
path = os.path.join(dir, wrapper)
if os.path.exists(path):
wrapper = path
defaults = {"SVN_SSH": wrapper}
os.environ.update(defaults)
raw = config.get("global", "svn-env")
if raw:
for line in raw.split("\n"):
env = line.strip()
if not env:
continue
try:
name, value = env.split("=", 1)
except ValueError:
sys.stderr.write("invalid svn environment line: %r\n" % env)
continue
os.environ[name] = value
def upload_binary(topdir, filename):
filepath = os.path.join(topdir, 'SOURCES', filename)
if not os.path.exists(filepath):
raise Error, "'%s' was not found" % filepath
sha1sum = file_hash(filepath)
if binary_exists(sha1sum):
return
host = config.get("binrepo", "upload_host")
upload_bin_helper = get_helper("upload-bin")
command = "ssh %s %s %s" % (host, upload_bin_helper, filename)
try:
filein = open(filepath, 'r')
except Error, e:
raise Error, "Could not open file %s\n" % filepath
def is_binary(path):
raw = config.get("binrepo", "upload-match",
"\.(7z|Z|bin|bz2|cpio|db|deb|egg|gem|gz|jar|jisp|lzma|"\
"pdf|pgn\\.gz|pk3|png|rpm|run|sdz|smzip|tar|tbz|"\
"tbz2|tgz|ttf|uqm|wad|war|xar|xpi|xz|zip|wav|mp3|ogg|"\
"jpg|png|gif|avi|mpg|mpeg|rar)$")
maxsize = config.getint("binrepo", "upload-match-size", "1048576") # 1MiB
expr = re.compile(raw)
name = os.path.basename(path)
if expr.search(name):
return True
st = os.stat(path)
if st[stat.ST_SIZE] >= maxsize:
return True
return False
def delete(paths, commit=False):
silent = config.get("log", "ignore-string", "SILENT")
for path in paths:
message = "%s: delete file %s" % (silent, path)
if binrepo.is_binary(path):
topdir = getpkgtopdir()
binrepo.update_sources(topdir, removed=[os.path.basename(path)])
if commit:
svn = SVN()
svn.commit(binrepo.sources_path(topdir), log=message)
else:
svn = SVN()
svn.remove(path, local=True)
if commit:
svn.commit(path, log=message)
def expand_group(group):
name, rev = layout.split_url_revision(group)
distro = None
if "/" in name:
distro, name = name.rsplit("/", 1)
found = config.get("submit-groups", name)
packages = [group]
if found:
packages = found.split()
if rev:
packages = [("%s@%s" % (package, rev))
for package in packages]
if distro:
packages = ["%s/%s" % (distro, package)
for package in packages]
return packages
def mapurl(url):
"""Maps a url following the regexp provided by the option url-map in
mgarepo.conf
"""
urlmap = config.get("global", "url-map")
newurl = url
if urlmap:
try:
expr_, replace = urlmap.split()[:2]
except ValueError:
sys.stderr.write("invalid url-map: %s" % urlmap)
else:
try:
newurl = re.sub(expr_, replace, url)
except re.error, errmsg:
sys.stderr.write("error in URL mapping regexp: %s" % errmsg)
def dump_file(releases, currentlog=None, template=None):
templpath = template or config.get("template", "path",
"/usr/share/mgarepo/default.chlog")
params = {}
if templpath is None or not os.path.exists(templpath):
params["source"] = default_template
sys.stderr.write("warning: %s not found. using built-in template.\n"%
templpath)
else:
params["file"] = templpath
releases_author = group_releases_by_author(releases)
revisions_author = group_revisions_by_author(currentlog)
params["searchList"] = [{"releases_by_author" : releases_author,
"releases" : releases,
"revisions_by_author": revisions_author}]
t = Template(**params)
return t.respond()
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
args.append("| less")
rawcmd = " ".join(args)
execcmd(rawcmd, show=True)
def main():
if not os.environ.has_key('REQUEST_METHOD'):
sys.stderr.write("error: this program is meant to be used as a cgi\n")
sys.exit(1)
print "Content-type: text/html\n\n"
try:
username = os.environ.get("REMOTE_USER")
method = os.environ.get("REQUEST_METHOD")
if not username or method != "POST":
show()
else:
useremail = config.get("users", username)
if not useremail:
raise CgiError, \
"Your email was not found. Contact the administrator!"
submit_packages(useremail)
except CgiError, e:
show(str(e), error=1)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
# extract the submit host from the svn host
type, rest = urllib.splittype(pkgdirurl)
host, path = urllib.splithost(rest)
user, host = urllib.splituser(host)
submithost, port = urllib.splitport(host)
del type, user, port, path, rest
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", submithost, createsrpm, "-t", target]
if not sid:
sid = uuid.uuid4()
define.append("sid=%s" % sid)
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side mgarepo versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
command = subprocess.list2cmdline(cmdargs)
status, output = execcmd(command)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.exit(status)
def _execsvn(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup")
if not kwargs.get("show") and args[0] not in localcmds:
args = list(args)
args.append("--non-interactive")
else:
kwargs["geterr"] = True
kwargs["cleanerr"] = True
if kwargs.get("xml"):
args.append("--xml")
self._set_env()
svn_command = config.get("global", "svn-command", "svn")
cmdstr = svn_command + " " + " ".join(args)
try:
return execcmd(cmdstr, **kwargs)
except Error, e:
msg = None
if e.args:
if "Permission denied" in e.args[0]:
msg = ("It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"https://wiki.mageia.org/en/Packagers_ssh"
" for more information.")
elif "authorization failed" in e.args[0]:
msg = ("Note that mgarepo does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error, "%s\n%s" % (e, msg)
raise
def dummy_wrapper(section, option=None, default=None, walk=False):
return config.get(section, option, default, wrap=False)
def make_handler():
uri = config.get("global", "ldap-uri")
if not uri:
server = config.get("global", "ldap-server")
if not server:
# ldap support is not enabled if ldap-uri nor ldap-server are
# defined
def dummy_wrapper(section, option=None, default=None, walk=False):
return config.get(section, option, default, wrap=False)
return dummy_wrapper
try:
port = int(config.get("global", "ldap-port", 389))
except ValueError:
raise Error, "the option ldap-port requires an integer, please "\
"check your configuration files"
uri = "ldap://%s:%d" % (server, port)
basedn = config.get("global", "ldap-base")
binddn = config.get("global", "ldap-binddn")
bindpw = config.get("global", "ldap-bindpw", "")
filterformat = config.get("global", "ldap-filterformat",
"(&(objectClass=inetOrgPerson)(uid=$username))", raw=1)
format = config.get("global", "ldap-resultformat", "$cn <$mail>", raw=1)
valid = {"yes": True, "no": False}
raw = config.get("global", "ldap-starttls", "no")
try:
starttls = valid[raw]
except KeyError:
raise Error, "invalid value %r for ldap-starttls, use "\
"'yes' or 'no'" % raw
try:
import ldap
except ImportError:
raise Error, "LDAP support needs the python-ldap package "\
"to be installed"
else:
from ldap.filter import escape_filter_chars
def users_wrapper(section, option=None, default=None, walk=False):
global users_cache
if walk:
raise Error, "ldapusers plugin does not support user listing"
assert option is not None, \
"When not section walking, option is required"
value = users_cache.get(option)
if value is not None:
return value
try:
l = ldap.initialize(uri)
if starttls:
l.start_tls_s()
if binddn:
l.bind(binddn, bindpw)
except ldap.LDAPError, e:
raise LDAPError(e)
try:
data = {"username": escape_filter_chars(option)}
filter = interpolate("ldap-filterformat", filterformat, data)
attrs = used_attributes(format)
try:
found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter,
attrlist=attrs)
except ldap.LDAPError, e:
raise LDAPError(e)
if found:
dn, entry = found[0]
entry = strip_entry(entry)
value = interpolate("ldap-resultformat", format, entry)
else:
# issue a warning?
value = config.get(section, option, default, wrap=False)
users_cache[option] = value
return value
def mirror_url():
mirror = config.get("global", "mirror")
return mirror
def layout_dirs():
devel_branch = config.get("global", "trunk-dir", "cauldron/")
devel_branch = os.path.normpath(devel_branch)
branches_dir = config.get("global", "branches-dir", "updates/")
branches_dir = os.path.normpath(branches_dir)
return devel_branch, branches_dir
def author_email(self, author):
return config.get("users", author)
def print_author_email(author):
email = config.get("users", author)
if not email:
raise Error, "author not found"
print email
