def layout_dirs():
devel_branch = config.get("global", "trunk-dir", "cauldron/")
devel_branch = os.path.normpath(devel_branch)
branches_dir = config.get("global", "branches-dir", "updates/")
branches_dir = os.path.normpath(branches_dir)
backports_dir = config.get("global", "backports-dir", "backports/")
backports_dir = os.path.normpath(backports_dir)
obsolete_dir = config.get("global", "obsolete-dir", "obsolete/")
obsolete_dir = os.path.normpath(obsolete_dir)
return devel_branch, branches_dir, backports_dir, obsolete_dir
def get_helper(name):
"""Tries to find the path of a helper script
It first looks if the helper has been explicitly defined in
configuration, if not, falls back to the default helper path, which can
also be defined in configuration file(s).
"""
helperdir = config.get("helper", "prefix", "/usr/share/repsys")
hpath = config.get("helper", name, None) or \
os.path.join(helperdir, name)
return hpath
def get_helper(name):
"""Tries to find the path of a helper script
It first looks if the helper has been explicitly defined in
configuration, if not, falls back to the default helper path, which can
also be defined in configuration file(s).
"""
helperdir = config.get("helper", "prefix", "/usr/share/repsys")
hpath = config.get("helper", name, None) or \
os.path.join(helperdir, name)
if not os.path.isfile(hpath):
log.warn("providing unexistent helper: %s", hpath)
return hpath
def repository_url(mirrored=False):
url = None
if mirrored and config.getbool("global", "use-mirror", "yes"):
url = config.get("global", "mirror")
if url is None:
url = config.get("global", "repository")
if not url:
# compatibility with the default_parent configuration option
default_parent = config.get("global", "default_parent")
if default_parent is None:
raise Error, "you need to set the 'repository' " \
"configuration option on repsys.conf"
url = convert_default_parent(default_parent)
return url
def submit_package(self, packageurl, packagerev, targetname):
username = os.environ.get("REMOTE_USER")
packager = config.get("users", username)
if not packager:
raise CgiError, "your email was not found"
elif not packagerev:
raise CgiError, "no revision provided"
elif not targetname:
raise CgiError, "no target provided"
else:
targetname = targetname.lower()
for target in get_targets():
if target.name.lower() == targetname:
break
else:
raise CgiError, "target not found"
try:
tmp = int(packagerev)
except ValueError:
raise CgiError, "invalid revision provided"
for allowed in target.allowed:
if packageurl.startswith(allowed):
break
else:
raise CgiError, "%s is not allowed for this target" \
% packageurl
get_srpm(packageurl,
revision=packagerev,
targetdirs=target.target,
packager=packager,
revname=1,
svnlog=1,
scripts=target.scripts)
return 1
def submit_package(self, packageurl, packagerev, targetname):
username = os.environ.get("REMOTE_USER")
packager = config.get("users", username)
if not packager:
raise CgiError, "your email was not found"
elif not packagerev:
raise CgiError, "no revision provided"
elif not targetname:
raise CgiError, "no target provided"
else:
targetname = targetname.lower()
for target in get_targets():
if target.name.lower() == targetname:
break
else:
raise CgiError, "target not found"
try:
tmp = int(packagerev)
except ValueError:
raise CgiError, "invalid revision provided"
for allowed in target.allowed:
if packageurl.startswith(allowed):
break
else:
raise CgiError, "%s is not allowed for this target" \
% packageurl
get_srpm(packageurl,
revision=packagerev,
targetdirs=target.target,
packager=packager,
revname=1,
svnlog=1,
scripts=target.scripts)
return 1
def download_sources(self):
f = open(join(self._git.path, "sources"))
downloader = config.get(
"global", "download-command",
"wget -c -O '$dest' $url").strip("'$dest' $url").split()
for line in f.readlines():
fields = line.split()
binurl = self.binrepo + "/" + self._package + "/"
if len(fields) == 2:
checksum, source = fields
binurl += source + "/"
else:
pattern = re.compile(r'^(.*) \((.*)\) = (.*)')
match = pattern.match(line)
hashtype = match.group(1)
source = match.group(2)
checksum = match.group(3)
binurl += source + "/" + hashtype.lower() + "/"
binurl += checksum + "/" + source
cmd = downloader + [join(self._git.path, source), binurl]
f.close()
status, output = execcmd(cmd, show=True)
if status == 0:
return True
else:
raise Error("Failed downloading %s, retcode: %d err: %s\n", status,
output)
def svn_log(pkgdirurl,
verbose=False,
limit=None,
revision=None,
releases=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl, releases=releases)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
pager = shlex.split(os.environ.get("PAGER", "less"))
p = subprocess.Popen(args, stdout=subprocess.PIPE)
p2 = subprocess.Popen(pager, stdin=p.stdout)
p2.wait()
p.wait()
else:
execcmd(args, show=True)
def get_old_log(pkgdirurl):
chlog = StringIO()
oldurl = config.get("log", "oldurl")
if oldurl:
svn = SVN(url=oldurl)
tmpdir = tempfile.mktemp()
try:
if oldurl == '.' or oldurl.startswith('./'):
pkgoldurl = os.path.join(pkgdirurl, oldurl)
else:
pkgname = layout.package_name(pkgdirurl)
pkgoldurl = os.path.join(svn.url, pkgname)
try:
# we're using HEAD here because fixes in misc/ (oldurl) may
# be newer than packages' last changed revision.
svn.export(pkgoldurl, tmpdir)
except Error:
pass
else:
logfile = os.path.join(tmpdir, "log")
if os.path.isfile(logfile):
with open(logfile, 'r', encoding='utf-8') as lf:
chlog.write("\n") # TODO needed?
log = lf.read()
log = escape_macros(log)
chlog.write(log)
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
chlog.seek(0)
return chlog
def get_old_log(pkgdirurl):
chlog = StringIO()
oldurl = config.get("log", "oldurl")
if oldurl:
svn = SVN()
tmpdir = tempfile.mktemp()
try:
pkgname = layout.package_name(pkgdirurl)
pkgoldurl = os.path.join(oldurl, pkgname)
try:
# we're using HEAD here because fixes in misc/ (oldurl) may
# be newer than packages' last changed revision.
svn.export(pkgoldurl, tmpdir)
except Error:
pass
else:
logfile = os.path.join(tmpdir, "log")
if os.path.isfile(logfile):
file = open(logfile)
chlog.write("\n") # TODO needed?
log = file.read()
log = escape_macros(log)
chlog.write(log)
file.close()
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
chlog.seek(0)
return chlog
def get_author_name(author):
found = emailpat.match(config.get("users", author, author))
gold = emailpat.match(usermap.get(author, ""))
name = ((found and found.group("name")) or (gold and gold.group("name"))
or author)
email = ((found and found.group("email")) or (gold and gold.group("email"))
or author + "@mageia.org")
return name, email
def get_repsys_cmd_path(cmd):
repsys = config.get("global", "repsys-cmd")
if repsys:
dir = os.path.dirname(repsys)
path = os.path.join(dir, cmd)
if os.path.exists(path):
return path
return cmd
def maintdb_get(package):
dlurl = config.get("maintdb", "url", "http://maintdb.mageia.org/")
dlurl = dlurl + "/" + package
h = httplib2.Http()
resp, content = h.request(dlurl, 'GET')
if resp.status != 200:
raise Exception('Package cannot be found in maintdb')
return content.rstrip('\n')
def __init__(self, package):
self._package = package
self._login = config.get("fedora", "login")
if self._login:
self._repourl = "ssh://" + self._login + "@pkgs.fedoraproject.org/rpms/" + self._package
else:
self._repourl = "https://src.fedoraproject.org/git/rpms/" + self._package + GIT.vcs_dirname
self._git = GIT(url=self._repourl)
def upload(paths, commit=False):
topdir = getpkgtopdir()
svn = detectVCS(topdir)
for path in paths:
if os.path.isdir(path) or binrepo.is_binary(path):
binrepo.upload_binary(topdir, os.path.basename(path))
binrepo.update_sources(topdir, added=[path])
if commit:
silent = config.get("log", "ignore-string", "SILENT")
message = "%s: new file %s" % (silent, path)
svn.commit(binrepo.sources_path(topdir), log=message)
else:
svn.add(path, local=True)
if commit:
silent = config.get("log", "ignore-string", "SILENT")
message = "%s: new file %s" % (silent, path)
svn.commit(path, log=message)
def __init__(self, path=None, url=None):
VCS.__init__(self, path, url)
vcs = getattr(VCS, "vcs")
vcs.append((self.vcs_name, self.vcs_dirname))
setattr(VCS, "vcs", vcs)
self.vcs_command = config.get("global", "git-command", ["git"])
self.vcs_supports['clone'] = True
self.env_defaults = {"GIT_SSH": self.vcs_wrapper}
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in repsys.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
args = ["ssh", host, createsrpm, "--list"]
execcmd(args, show=True)
sys.exit(0) # it is invoked via optparse callback, thus we need to
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in repsys.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
command = "ssh %s %s --list" % (host, createsrpm)
execcmd(command, show=True)
sys.exit(0)
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in repsys.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
command = "ssh %s %s --list" % (host, createsrpm)
execcmd(command, show=True)
sys.exit(0)
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error("no submit host defined in repsys.conf")
createsrpm = get_helper("create-srpm")
#TODO make it configurable
args = ["ssh", host, createsrpm, "--list"]
execcmd(args, show=true)
sys.exit(0) # it is invoked via optparse callback, thus we need to
def default_parent(url):
if url.find("://") == -1:
default_parent = config.get("global", "default_parent")
if not default_parent:
raise Error, "received a relative url, " \
"but default_parent was not setup"
parsed = list(urlparse.urlparse(default_parent))
parsed[2] = os.path.normpath(parsed[2] + "/" + url)
url = urlparse.urlunparse(parsed)
return url
def default_parent(url):
if url.find("://") == -1:
default_parent = config.get("global", "default_parent")
if not default_parent:
raise Error, "received a relative url, " \
"but default_parent was not setup"
parsed = list(urlparse.urlparse(default_parent))
parsed[2] = os.path.normpath(parsed[2] + "/" + url)
url = urlparse.urlunparse(parsed)
return url
def get_auth(username=None, password=None):
set_username = 1
set_password = 1
if not username:
username = config.get("auth", "username")
if not username:
username = raw_input("username: "******"auth", "password")
if not password:
password = getpass.getpass("password: "******"auth", "username", username)
if set_password:
config.set("auth", "password", password)
return username, password
def get_auth(username=None, password=None):
set_username = 1
set_password = 1
if not username:
username = config.get("auth", "username")
if not username:
username = raw_input("username: "******"auth", "password")
if not password:
password = getpass.getpass("password: "******"auth", "username", username)
if set_password:
config.set("auth", "password", password)
return username, password
def download_binary(topdir, sha1, filename):
fmt = config.get("global", "download-command", "wget -c -O '$dest' $url")
url = config.get("binrepo", "download_url", "http://binrepo.mageia.org/")
url = mirror.normalize_path(url + "/" + sha1)
dest = os.path.join(topdir, 'SOURCES', filename)
if os.path.exists(dest):
if file_hash(dest) == sha1:
return 1
else:
raise Error("File with incorrect sha1sum: %s" % dest)
context = {"dest": dest, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError as e:
raise Error("invalid variable %r in download-command "\
"configuration option" % e)
try:
status, output = execcmd(cmd, show=True)
except Error as e:
os.unlink(dest)
raise Error("Could not download file %s\n" % url)
def filter_log_lines(lines):
# Lines in commit messages beginning with CLOG will be the only shown
# in the changelog. These lines will have the CLOG token and blanks
# stripped from the beginning.
onlylines = None
clogstr = config.get("log", "unignore-string")
if clogstr:
clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr)
onlylines = [clogre.sub("", line)
for line in lines if line.startswith(clogstr)]
if onlylines:
filtered = onlylines
else:
# Lines in commit messages containing SILENT at any position will be
# skipped; commits with their log messages beggining with SILENT in the
# first positionj of the first line will have all lines ignored.
ignstr = config.get("log", "ignore-string", "SILENT")
if len(lines) and lines[0].startswith(ignstr):
return []
filtered = [line for line in lines if ignstr not in line]
return filtered
def _set_env(self):
wrapper = "repsys-ssh"
repsys = config.get("global", "repsys-cmd")
if repsys:
dir = os.path.dirname(repsys)
path = os.path.join(dir, wrapper)
if os.path.exists(path):
wrapper = path
defaults = {"SVN_SSH": wrapper}
os.environ.update(defaults)
raw = config.get("global", "svn-env")
if raw:
for line in raw.split("\n"):
env = line.strip()
if not env:
continue
try:
name, value = env.split("=", 1)
except ValueError:
sys.stderr.write("invalid svn environment line: %r\n" % env)
continue
os.environ[name] = value
def filter_log_lines(lines):
# Lines in commit messages beginning with CLOG will be the only shown
# in the changelog. These lines will have the CLOG token and blanks
# stripped from the beginning.
onlylines = None
clogstr = config.get("log", "unignore-string")
if clogstr:
clogre = re.compile(r"(^%s[^ \t]?[ \t])" % clogstr)
onlylines = [
clogre.sub("", line) for line in lines if line.startswith(clogstr)
]
if onlylines:
filtered = onlylines
else:
# Lines in commit messages containing SILENT at any position will be
# skipped; commits with their log messages beggining with SILENT in the
# first positionj of the first line will have all lines ignored.
ignstr = config.get("log", "ignore-string", "SILENT")
if len(lines) and lines[0].startswith(ignstr):
return []
filtered = [line for line in lines if ignstr not in line]
return filtered
def _set_env(self):
wrapper = "repsys-ssh"
repsys = config.get("global", "repsys-cmd")
if repsys:
dir = os.path.dirname(repsys)
path = os.path.join(dir, wrapper)
if os.path.exists(path):
wrapper = path
defaults = {"SVN_SSH": wrapper}
os.environ.update(defaults)
raw = config.get("global", "svn-env")
if raw:
for line in raw.split("\n"):
env = line.strip()
if not env:
continue
try:
name, value = env.split("=", 1)
except ValueError:
sys.stderr.write("invalid svn environment line: %r\n" % env)
continue
os.environ[name] = value
def expand_group(group):
name, rev = layout.split_url_revision(group)
distro = None
if "/" in name:
distro, name = name.rsplit("/", 1)
found = config.get("submit-groups", name)
packages = [group]
if found:
packages = found.split()
if rev:
packages = [("%s@%s" % (package, rev)) for package in packages]
if distro:
packages = ["%s/%s" % (distro, package) for package in packages]
return packages
def delete(paths, commit=False):
silent = config.get("log", "ignore-string", "SILENT")
topdir = getpkgtopdir()
svn = detectVCS(topdir)
for path in paths:
message = "%s: delete file %s" % (silent, path)
if binrepo.is_binary(path):
binrepo.update_sources(topdir, removed=[os.path.basename(path)])
if commit:
svn.commit(binrepo.sources_path(topdir), log=message)
else:
svn.remove(path, local=True)
if commit:
svn.commit(path, log=message)
def expand_group(group):
name, rev = layout.split_url_revision(group)
distro = None
if "/" in name:
distro, name = name.rsplit("/", 1)
found = config.get("submit-groups", name)
packages = [group]
if found:
packages = found.split()
if rev:
packages = [("%s@%s" % (package, rev))
for package in packages]
if distro:
packages = ["%s/%s" % (distro, package)
for package in packages]
return packages
def mapurl(url):
"""Maps a url following the regexp provided by the option url-map in
repsys.conf
"""
urlmap = config.get("global", "url-map")
newurl = url
if urlmap:
try:
expr_, replace = urlmap.split()[:2]
except ValueError:
log.error("invalid url-map: %s", urlmap)
else:
try:
newurl = re.sub(expr_, replace, url)
except re.error, errmsg:
log.error("error in URL mapping regexp: %s", errmsg)
def mapurl(url):
"""Maps a url following the regexp provided by the option url-map in
repsys.conf
"""
urlmap = config.get("global", "url-map")
newurl = url
if urlmap:
try:
expr_, replace = urlmap.split()[:2]
except ValueError:
log.error("invalid url-map: %s", urlmap)
else:
try:
newurl = re.sub(expr_, replace, url)
except re.error, errmsg:
log.error("error in URL mapping regexp: %s", errmsg)
def _set_env(self):
os.environ.update(self.env_defaults)
if self.vcs_name:
raw = config.get("global", "%s-env" % self.vcs_name)
if raw:
for line in raw.split("\n"):
env = line.strip()
if not env:
continue
try:
name, value = env.split("=", 1)
except ValueError:
sys.stderr.write("invalid %s environment line: %r\n" %
(self.vcs_name, env))
continue
os.environ[name] = value
def build_rpm(build_cmd="a",
verbose=False,
rpmlint=True,
packager = "",
macros = []):
top = os.getcwdu()
topdir = "--define '_topdir %s'" % top
builddir = "--define '_builddir %s/%s'" % (top, "BUILD")
rpmdir = "--define '_rpmdir %s/%s'" % (top, "RPMS")
sourcedir = "--define '_sourcedir %s/%s'" % (top, "SOURCES")
specdir = "--define '_specdir %s/%s'" % (top, "SPECS")
srcrpmdir = "--define '_srcrpmdir %s/%s'" % (top, "SRPMS")
patchdir = "--define '_patchdir %s/%s'" % (top, "SOURCES")
build = os.path.join(top, "BUILD")
if not os.path.exists(build):
os.mkdir(build)
specsdir = os.path.join(top, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if packager:
packager = " --define 'packager %s'" % packager
defs = rpm_macros_defs(macros)
rpmbuild = config.get("helper", "rpmbuild", "rpmbuild")
begintime = time()
execcmd("LC_ALL=C %s -b%s %s %s %s %s %s %s %s %s %s %s" %
(rpmbuild, build_cmd, topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, packager, spec, defs), show=verbose)
endtime = time()
if rpmlint:
status, output = execcmd("rpm %s %s --define 'debug_package_and_restore %%{debug_package}' --define '_query_all_fmt %%{_srcrpmdir}/%%{SOURCERPM} %%{_rpmdir}/%%{_build_name_fmt}' -q --specfile %s" % (rpmdir, srcrpmdir, spec))
checklist = []
for ps in output.split("\n"):
for p in ps.split():
if p not in checklist and os.path.exists(p):
# We assume that only packages with matching filename and
# that has been built since we invoked rpmbuild belongs to
# this build and should be checked
stat = os.stat(p)
if stat.st_ctime > begintime and stat.st_ctime < endtime:
checklist.append(p)
if checklist:
execcmd("rpmlint %s" % string.join(checklist), show=True)
def build_rpm(build_cmd="a",
verbose=False,
rpmlint=True,
packager = "",
macros = []):
top = os.getcwdu()
topdir = "--define '_topdir %s'" % top
builddir = "--define '_builddir %s/%s'" % (top, "BUILD")
rpmdir = "--define '_rpmdir %s/%s'" % (top, "RPMS")
sourcedir = "--define '_sourcedir %s/%s'" % (top, "SOURCES")
specdir = "--define '_specdir %s/%s'" % (top, "SPECS")
srcrpmdir = "--define '_srcrpmdir %s/%s'" % (top, "SRPMS")
patchdir = "--define '_patchdir %s/%s'" % (top, "SOURCES")
build = os.path.join(top, "BUILD")
if not os.path.exists(build):
os.mkdir(build)
specsdir = os.path.join(top, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if packager:
packager = " --define 'packager %s'" % packager
defs = rpm_macros_defs(macros)
rpmbuild = config.get("helper", "rpmbuild", "rpmbuild")
begintime = time()
execcmd("LC_ALL=C %s -b%s %s %s %s %s %s %s %s %s %s %s" %
(rpmbuild, build_cmd, topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, packager, spec, defs), show=verbose)
endtime = time()
if rpmlint:
status, output = execcmd("rpm %s %s --define 'debug_package_and_restore %%{debug_package}' --define '_query_all_fmt %%{_srcrpmdir}/%%{SOURCERPM} %%{_rpmdir}/%%{_build_name_fmt}' -q --specfile %s" % (rpmdir, srcrpmdir, spec))
checklist = []
for ps in output.split("\n"):
for p in ps.split():
if p not in checklist and os.path.exists(p):
# We assume that only packages with matching filename and
# that has been built since we invoked rpmbuild belongs to
# this build and should be checked
stat = os.stat(p)
if stat.st_ctime > begintime and stat.st_ctime < endtime:
checklist.append(p)
if checklist:
execcmd("rpmlint %s" % string.join(checklist), show=True)
def dump_file(releases, currentlog=None, template=None):
templpath = template or config.get("template", "path",
"/usr/share/repsys/default.chlog")
params = {}
if templpath is None or not os.path.exists(templpath):
params["source"] = default_template
sys.stderr.write("warning: %s not found. using built-in template.\n"%
templpath)
else:
params["file"] = templpath
releases_author = group_releases_by_author(releases)
revisions_author = group_revisions_by_author(currentlog)
params["searchList"] = [{"releases_by_author" : releases_author,
"releases" : releases,
"revisions_by_author": revisions_author}]
t = Template(**params)
return t.respond()
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
args.append("| less")
rawcmd = " ".join(args)
execcmd(rawcmd, show=True)
def submit(urls,
target,
define=[],
submithost=None,
port=None,
atonce=False,
debug=False,
keeplog=False):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
raise Error, "no submit host defined in configuration"
if port is None:
port = config.getint("submit", "port", "22")
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", "-p", str(port), submithost, createsrpm, "-t", target]
if debug:
baseargs.append("--debug")
if keeplog:
baseargs.append("--keeplog")
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
status, output = execcmd(cmdargs, noerror=1)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.stderr.write("\n")
sys.exit(status)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
args.append("| less")
rawcmd = " ".join(args)
execcmd(rawcmd, show=True)
def users_wrapper(section, option=None, default=None, walk=False):
global users_cache
if walk:
raise Error("ldapusers plugin does not support user listing")
assert option is not None, \
"When not section walking, option is required"
value = users_cache.get(option)
if value is not None:
return value
try:
#l = ldap.initialize(uri)
l = Connection(uri, auto_bind=True, use_tls=starttls)
# if starttls:
# l.start_tls() #l.start_tls_s()
if binddn:
l.bind(binddn, bindpw)
except ldap3.LDAPExceptionError as e:
raise LDAPError(e)
try:
data = {"username": escape_filter_chars(option)}
filter = interpolate("ldap-filterformat", filterformat, data)
attrs = used_attributes(format)
try:
found = l.search(search_base='o='.basedn,
search_filter=filter,
attributes=attrs,
search_scope=SUBTREE)
# found = l.search_s(basedn, ldap.SCOPE_SUBTREE, filter,
# attrlist=attrs)
except ldap3.LDAPError as e:
raise LDAPError(e)
if found:
dn, entry = found[0]
entry = strip_entry(entry)
value = interpolate("ldap-resultformat", format, entry)
else:
# issue a warning?
value = config.get(section, option, default, wrap=False)
users_cache[option] = value
return value
finally:
l.unbind()
def _execsvn(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup")
collecterr = False
if kwargs.get("show"):
if not kwargs.get("local"):
collecterr = True
else:
if args[0] not in localcmds:
args = list(args)
args.append("--non-interactive")
cleanerr = True
self._set_env()
svn_command = config.get("global", "svn-command", "svn")
cmdargs = [svn_command]
cmdargs.extend(args)
try:
return execcmd(cmdargs,
show=kwargs.get("show", False),
noerror=kwargs.get("noerror", False),
collecterr=collecterr,
cleanerr=cleanerr)
except Error, e:
msg = None
if e.args:
if "Permission denied" in e.args[0]:
msg = (
"It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration"
" for more information.")
elif "authorization failed" in e.args[0]:
msg = ("Note that repsys does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error, "%s\n%s" % (e, msg)
raise
def upload_binary(topdir, filename):
filepath = os.path.join(topdir, 'SOURCES', filename)
if not os.path.exists(filepath):
raise Error("'%s' was not found" % filepath)
sha1sum = file_hash(filepath)
if binary_exists(sha1sum):
return
host = config.get("binrepo", "upload_host")
upload_bin_helper = get_helper("upload-bin")
command = ["ssh", host, upload_bin_helper, filename]
try:
filein = open(filepath, 'r')
except Error as e:
raise Error("Could not open file %s\n" % filepath)
status, output = execcmd(*command,
show=True,
collecterr=True,
stdin=filein)
def main():
if not os.environ.has_key('REQUEST_METHOD'):
sys.stderr.write("error: this program is meant to be used as a cgi\n")
sys.exit(1)
print "Content-type: text/html\n\n"
try:
username = os.environ.get("REMOTE_USER")
method = os.environ.get("REQUEST_METHOD")
if not username or method != "POST":
show()
else:
useremail = config.get("users", username)
if not useremail:
raise CgiError, \
"Your email was not found. Contact the administrator!"
submit_packages(useremail)
except CgiError, e:
show(str(e), error=1)
def _execsvn(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup")
collecterr = False
if kwargs.get("show"):
if not kwargs.get("local"):
collecterr = True
else:
if args[0] not in localcmds:
args = list(args)
args.append("--non-interactive")
cleanerr = True
self._set_env()
svn_command = config.get("global", "svn-command", "svn")
cmdargs = [svn_command]
cmdargs.extend(args)
try:
return execcmd(cmdargs, show=kwargs.get("show", False),
noerror=kwargs.get("noerror", False),
collecterr=collecterr, cleanerr=cleanerr)
except Error, e:
msg = None
if e.args:
if "Permission denied" in e.args[0]:
msg = ("It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration"
" for more information.")
elif "authorization failed" in e.args[0]:
msg = ("Note that repsys does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error, "%s\n%s" % (e, msg)
raise
def submit(urls, target, define=[], submithost=None, port=None,
atonce=False, debug=False, keeplog=False):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
raise Error, "no submit host defined in configuration"
if port is None:
port = config.getint("submit", "port", "22")
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", "-p", str(port), submithost, createsrpm,
"-t", target]
if debug:
baseargs.append("--debug")
if keeplog:
baseargs.append("--keeplog")
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
status, output = execcmd(cmdargs, noerror=1)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.stderr.write("\n")
sys.exit(status)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(str(limit))
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
pager = shlex.split(os.environ.get("PAGER", "less"))
p = subprocess.Popen(args, stdout=subprocess.PIPE)
p2 = subprocess.Popen(pager, stdin=p.stdout)
p2.wait()
p.wait()
else:
execcmd(args, show=True)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
# extract the submit host from the svn host
type, rest = urllib.splittype(pkgdirurl)
host, path = urllib.splithost(rest)
user, host = urllib.splituser(host)
submithost, port = urllib.splitport(host)
del type, user, port, path, rest
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", submithost, createsrpm, "-t", target]
if not sid:
sid = uuid.uuid4()
define.append("sid=%s" % sid)
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
command = subprocess.list2cmdline(cmdargs)
status, output = execcmd(command)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.exit(status)
def svn2rpm(pkgdirurl, rev=None, size=None, submit=False,
template=None, macros=[], exported=None):
concat = config.get("log", "concat", "").split()
revoffset = get_revision_offset()
svn = SVN()
pkgreleasesurl = layout.checkout_url(pkgdirurl, releases=True)
pkgcurrenturl = layout.checkout_url(pkgdirurl)
releaseslog = svn.log(pkgreleasesurl, noerror=1)
currentlog = svn.log(pkgcurrenturl, limit=size, start=rev,
end=revoffset)
# sort releases by copyfrom-revision, so that markreleases for same
# revisions won't look empty
releasesdata = []
if releaseslog:
for relentry in releaseslog[::-1]:
try:
(version, release, relrevision) = \
parse_markrelease_log(relentry)
except InvalidEntryError:
continue
releasesdata.append((relrevision, -relentry.revision, relentry,
version, release))
releasesdata.sort()
# collect valid releases using the versions provided by the changes and
# the packages
prevrevision = 0
releases = []
for (relrevision, dummy, relentry, version, release) in releasesdata:
if prevrevision == relrevision:
# ignore older markrelease of the same revision, since they
# will have no history
continue
entries = [entry for entry in currentlog
if relrevision >= entry.revision and
(prevrevision < entry.revision)]
if not entries:
#XXX probably a forced release, without commits in current/,
# check if this is the right behavior
sys.stderr.write("warning: skipping (possible) release "
"%s-%s@%s, no commits since previous markrelease (r%r)\n" %
(version, release, relrevision, prevrevision))
continue
release = make_release(author=relentry.author,
revision=relentry.revision, date=relentry.date,
lines=relentry.lines, entries=entries,
version=version, release=release)
releases.append(release)
prevrevision = relrevision
# look for commits that have been not submitted (released) yet
# this is done by getting all log entries newer (greater revision no.)
# than releasesdata[-1] (in the case it exists)
if releasesdata:
latest_revision = releasesdata[-1][0] # the latest copied rev
else:
latest_revision = 0
notsubmitted = [entry for entry in currentlog
if entry.revision > latest_revision]
if notsubmitted:
# if they are not submitted yet, what we have to do is to add
# a release/version number from getrelease()
version, release = getrelease(pkgdirurl, macros=macros,
exported=exported)
toprelease = make_release(entries=notsubmitted, released=False,
version=version, release=release)
releases.append(toprelease)
data = dump_file(releases[::-1], currentlog=currentlog, template=template)
return data
def __init__(self):
VCS.__init__(self)
self.vcs_name = "git"
self.vcs_command = config.get("global", "git-command", "git")
self.vcs_supports['clone'] = True
self.env_defaults = {"GIT_SSH": self.vcs_wrapper}
def author_email(self, author):
return config.get("users", author)
def mirror_url():
mirror = config.get("global", "mirror")
return mirror
def layout_dirs():
devel_branch = config.get("global", "trunk-dir", "cooker/")
devel_branch = os.path.normpath(devel_branch)
branches_dir = config.get("global", "branches-dir", "updates/")
branches_dir = os.path.normpath(branches_dir)
return devel_branch, branches_dir
def get_author_name(author):
found = emailpat.match(config.get("users", author, author))
name = ((found and found.group("name")) or author)
email = ((found and found.group("email")) or author)
return name, email
def get_srpm(pkgdirurl,
mode = "current",
targetdirs = None,
version = None,
release = None,
revision = None,
packager = "",
revname = 0,
svnlog = 0,
scripts = [],
submit = False,
template = None,
distro = None,
macros = [],
verbose = 0,
strict = False):
svn = SVN()
tmpdir = tempfile.mktemp()
topdir = "_topdir %s" % tmpdir
builddir = "_builddir %s/%s" % (tmpdir, "BUILD")
rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS")
sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES")
specdir = "_specdir %s/%s" % (tmpdir, "SPECS")
srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS")
patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES")
temppath = "_tmppath %s" % (tmpdir)
rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir,
sourcedir, specdir, srcrpmdir, patchdir, temppath)]
try:
if mode == "version":
geturl = layout.checkout_url(pkgdirurl, version=version,
release=release)
elif mode == "pristine":
geturl = layout.checkout_url(pkgdirurl, pristine=True)
elif mode == "current" or mode == "revision":
#FIXME we should handle revisions specified using @REV
geturl = layout.checkout_url(pkgdirurl)
else:
raise Error, "unsupported get_srpm mode: %s" % mode
strict = strict or config.getbool("submit", "strict-revision", False)
if strict and not rev_touched_url(geturl, revision):
#FIXME would be nice to have the revision number even when
# revision is None
raise Error, "the revision %s does not change anything "\
"inside %s" % (revision or "HEAD", geturl)
mirror.info(geturl)
svn.export(geturl, tmpdir, rev=revision)
srpmsdir = os.path.join(tmpdir, "SRPMS")
os.mkdir(srpmsdir)
specsdir = os.path.join(tmpdir, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if config.getbool("srpm", "run-prep", False):
makefile = os.path.join(tmpdir, "Makefile")
if os.path.exists(makefile):
execcmd(("make", "-C", tmpdir, "srpm-prep"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if svnlog:
submit = not not revision
log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
template=template, macros=macros, exported=tmpdir)
for script in scripts:
#FIXME revision can be "None"
status, output = execcmd(script, tmpdir, spec, str(revision),
noerror=1)
if status != 0:
raise Error, "script %s failed" % script
if packager:
packager = " --define 'packager %s'" % packager
if distro:
cmpdistro = distro.lower()
for target in cgiutil.get_targets():
if target.name.lower() == cmpdistro:
macros.extend(target.macros)
break
else:
raise Error, "no such submit target in configuration: %s" % (distro)
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
args = [sourcecmd, "-bs", "--nodeps"]
for pair in rpmdefs:
args.extend(pair)
for pair in macros:
args.extend(("--define", "%s %s" % pair))
args.append(spec)
try:
execcmd(args)
except CommandError, e:
if config.getbool("global", "verbose"):
cmdline = e.cmdline + "\n"
else:
cmdline = ""
raise Error, ("error while creating the source RPM "
"(with %s):\n%s%s" % (sourcecmd, cmdline, e.output))
# copy the generated SRPMs to their target locations
targetsrpms = []
urlrev = None
if revname:
urlrev = revision or layout.get_url_revision(geturl)
if not targetdirs:
targetdirs = (".",)
srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
if not srpms:
# something fishy happened
raise Error, "no SRPMS were found at %s" % srpmsdir
for srpm in srpms:
name = os.path.basename(srpm)
if revname:
name = "@%s:%s" % (urlrev, name)
for targetdir in targetdirs:
newpath = os.path.join(targetdir, name)
targetsrpms.append(newpath)
if os.path.exists(newpath):
# should we warn?
os.unlink(newpath)
shutil.copy(srpm, newpath)
if verbose:
sys.stderr.write("Wrote: %s\n" % newpath)
return targetsrpms
def dummy_wrapper(section, option=None, default=None, walk=False):
return config.get(section, option, default, wrap=False)
def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
baseurl=None, baseold=None, logmsg=None, rename=True):
svn = SVN()
srpm = SRPM(srpmfile)
tmpdir = tempfile.mktemp()
if baseurl:
pkgurl = mirror._joinurl(baseurl, srpm.name)
else:
pkgurl = layout.package_url(srpm.name, distro=branch,
mirrored=False)
print "Importing package to %s" % pkgurl
try:
if srpm.epoch:
version = "%s:%s" % (srpm.epoch, srpm.version)
else:
version = srpm.version
versionurl = "/".join([pkgurl, "releases", version])
releaseurl = "/".join([versionurl, srpm.release])
currenturl = "/".join([pkgurl, "current"])
currentdir = os.path.join(tmpdir, "current")
#FIXME when pre-commit hook fails, there's no clear way to know
# what happened
ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory")
if ret or not svn.ls(currenturl, noerror=1):
svn.checkout(pkgurl, tmpdir)
svn.mkdir(os.path.join(tmpdir, "releases"))
svn.mkdir(currentdir)
svn.mkdir(os.path.join(currentdir, "SPECS"))
svn.mkdir(os.path.join(currentdir, "SOURCES"))
#svn.commit(tmpdir,log="Created package structure.")
version_exists = 1
else:
if svn.ls(releaseurl, noerror=1):
raise Error, "release already exists"
svn.checkout("/".join([pkgurl, "current"]), tmpdir)
svn.mkdir(versionurl, noerror=1,
log="Created directory for version %s." % version)
currentdir = tmpdir
specsdir = os.path.join(currentdir, "SPECS")
sourcesdir = os.path.join(currentdir, "SOURCES")
unpackdir = tempfile.mktemp()
os.mkdir(unpackdir)
try:
srpm.unpack(unpackdir)
uspecsdir = os.path.join(unpackdir, "SPECS")
usourcesdir = os.path.join(unpackdir, "SOURCES")
uspecsentries = os.listdir(uspecsdir)
usourcesentries = os.listdir(usourcesdir)
specsentries = os.listdir(specsdir)
sourcesentries = os.listdir(sourcesdir)
# Remove old entries
for entry in [x for x in specsentries
if x not in uspecsentries]:
if entry == ".svn":
continue
entrypath = os.path.join(specsdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
for entry in [x for x in sourcesentries
if x not in usourcesentries]:
if entry == ".svn":
continue
entrypath = os.path.join(sourcesdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
# Copy all files
execcmd(["cp", "-rf", uspecsdir, currentdir])
execcmd(["cp", "-rf", usourcesdir, currentdir])
# Add new entries
for entry in [x for x in uspecsentries
if x not in specsentries]:
entrypath = os.path.join(specsdir, entry)
svn.add(entrypath)
for entry in [x for x in usourcesentries
if x not in sourcesentries]:
entrypath = os.path.join(sourcesdir, entry)
svn.add(entrypath)
finally:
if os.path.isdir(unpackdir):
shutil.rmtree(unpackdir)
specs = glob.glob(os.path.join(specsdir, "*.spec"))
if not specs:
raise Error, "no spec file found on %s" % specsdir
if len(specs) > 1:
raise Error, "more than one spec file found on %s" % specsdir
specpath = specs[0]
if rename:
specfile = os.path.basename(specpath)
specname = specfile[:-len(".spec")]
if specname != srpm.name:
newname = srpm.name + ".spec"
newpath = os.path.join(specsdir, newname)
sys.stderr.write("warning: renaming spec file to '%s' "
"(use -n to disable it)\n" % (newname))
os.rename(specpath, newpath)
try:
svn.remove(specpath)
except Error:
# file not tracked
svn.revert(specpath)
svn.add(newpath)
specpath = newpath
if striplog:
specpath = specpath
fspec = open(specpath)
spec, chlog = log.split_spec_changelog(fspec)
fspec.close()
fspec = open(specpath, "w")
fspec.writelines(spec)
fspec.close()
chlog.seek(0, os.SEEK_END)
if chlog.tell() != 0:
chlog.seek(0)
#FIXME move it to layout.py
oldurl = baseold or config.get("log", "oldurl")
pkgoldurl = mirror._joinurl(oldurl, srpm.name)
svn.mkdir(pkgoldurl, noerror=1,
log="created old log directory for %s" % srpm.name)
logtmp = tempfile.mktemp()
try:
svn.checkout(pkgoldurl, logtmp)
miscpath = os.path.join(logtmp, "log")
fmisc = open(miscpath, "w+")
fmisc.writelines(chlog)
fmisc.close()
svn.add(miscpath)
svn.commit(logtmp,
log="imported old log for %s" % srpm.name)
finally:
if os.path.isdir(logtmp):
shutil.rmtree(logtmp)
svn.commit(tmpdir,
log=logmsg or ("imported package %s" % srpm.name))
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
# Do revision and pristine tag copies
pristineurl = layout.checkout_url(pkgurl, pristine=True)
svn.remove(pristineurl, noerror=1,
log="Removing previous pristine/ directory.")
currenturl = layout.checkout_url(pkgurl)
svn.copy(currenturl, pristineurl,
log="Copying release %s-%s to pristine/ directory." %
(version, srpm.release))
if markrelease:
svn.copy(currenturl, releaseurl,
log="Copying release %s-%s to releases/ directory." %
(version, srpm.release))
if specstl:
specst, _ = specstl[0]
if specst == "?":
toadd.append(specpath)
# add source files:
for source, url in sources.iteritems():
sourcepath = os.path.join(sourcesdir, source)
pst = sourcesst.get(source)
if pst:
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
sys.stderr.write("warning: %s not found, skipping\n" % sourcepath)
elif download and not os.path.isfile(sourcepath):
print "%s not found, downloading from %s" % (sourcepath, url)
fmt = config.get("global", "download-command",
"wget -c -O '$dest' $url")
context = {"dest": sourcepath, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError, e:
raise Error, "invalid variable %r in download-command "\
"configuration option" % e
execcmd(cmd, show=True)
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
raise Error, "file not found: %s" % sourcepath
# rm entries not found in sources and still in svn
found = os.listdir(sourcesdir)
toremove = []
for entry in found:
