def svn_log(pkgdirurl,
verbose=False,
limit=None,
revision=None,
releases=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl, releases=releases)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
pager = shlex.split(os.environ.get("PAGER", "less"))
p = subprocess.Popen(args, stdout=subprocess.PIPE)
p2 = subprocess.Popen(pager, stdin=p.stdout)
p2.wait()
p.wait()
else:
execcmd(args, show=True)
def clone(self, url, targetpath, **kwargs):
if url.split(':')[0].find("svn") < 0:
return VCS.clone(self, url, targetpath, **kwargs)
else:
# To speed things up on huge repositories, we'll just grab all the
# revision numbers for this specific directory and grab these only
# in stead of having to go through each and every revision...
retval, result = execcmd("svn log --stop-on-copy --xml %s" % url)
if retval:
return retval
parser = ElementTree.XMLTreeBuilder()
result = "".join(result.split("\n"))
parser.feed(result)
log = parser.close()
logentries = log.getiterator("logentry")
revisions = []
topurl = dirname(url)
trunk = basename(url)
tags = "releases"
execcmd("git svn init %s --trunk=%s --tags=%s %s" % (topurl, trunk, tags, targetpath), show=True)
chdir(targetpath)
for entry in logentries:
revisions.append(entry.attrib["revision"])
while revisions:
execcmd("git svn fetch -r%d" % int(revisions.pop()), show=True)
cmd = ["svn", "rebase"]
return self._execVcs_success(*cmd, **kwargs)
def clone(self, url, targetpath, **kwargs):
if url.split(':')[0].find("svn") < 0:
return VCS.clone(self, url, targetpath, **kwargs)
else:
# To speed things up on huge repositories, we'll just grab all the
# revision numbers for this specific directory and grab these only
# in stead of having to go through each and every revision...
retval, result = execcmd("svn log --stop-on-copy --xml %s" % url)
if retval:
return retval
parser = ElementTree.XMLTreeBuilder()
result = "".join(result.split("\n"))
parser.feed(result)
log = parser.close()
logentries = log.getiterator("logentry")
revisions = []
topurl = dirname(url)
trunk = basename(url)
tags = "releases"
execcmd("git svn init %s --trunk=%s --tags=%s %s" %
(topurl, trunk, tags, targetpath),
show=True)
chdir(targetpath)
for entry in logentries:
revisions.append(entry.attrib["revision"])
while revisions:
execcmd("git svn fetch -r%d" % int(revisions.pop()), show=True)
cmd = ["svn", "rebase"]
return self._execVcs_success(*cmd, **kwargs)
def unpack(self, topdir):
args = [
"rpm", "-i", "--nodeps", "--define",
"_sourcedir {0}/SOURCES".format(topdir), "--define",
"_specdir {0}/SPECS".format(topdir), "--define",
"_patchdir {0}/SOURCES".format(topdir), self.filename
]
execcmd(args)
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error("no submit host defined in repsys.conf")
createsrpm = get_helper("create-srpm")
#TODO make it configurable
args = ["ssh", host, createsrpm, "--list"]
execcmd(args, show=true)
sys.exit(0) # it is invoked via optparse callback, thus we need to
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in repsys.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
command = "ssh %s %s --list" % (host, createsrpm)
execcmd(command, show=True)
sys.exit(0)
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in repsys.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
command = "ssh %s %s --list" % (host, createsrpm)
execcmd(command, show=True)
sys.exit(0)
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in repsys.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
args = ["ssh", host, createsrpm, "--list"]
execcmd(args, show=True)
sys.exit(0) # it is invoked via optparse callback, thus we need to
def _execVcs(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup")
if not kwargs.get("show") and args[0] not in localcmds:
args = list(args)
args.append("--non-interactive")
else:
kwargs["geterr"] = True
kwargs["cleanerr"] = True
cmdstr = self.vcs_command + " " + " ".join(args)
try:
return execcmd(cmdstr, **kwargs)
except Error, e:
msg = None
if e.args:
if "Permission denied" in e.args[0]:
msg = ("It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration"
" for more information.")
elif "authorization failed" in e.args[0]:
msg = ("Note that repsys does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error, "%s\n%s" % (e, msg)
raise
def _getinfo(self):
args = ["rpm", "-qp", "--qf", "%{name} %{epoch} %{release} %{version}",
self.filename]
status, output = execcmd(args)
self.name, self.epoch, self.release, self.version = output.split()
if self.epoch == "(none)":
self.epoch = None
def download_sources(self):
f = open(join(self._git.path, "sources"))
downloader = config.get(
"global", "download-command",
"wget -c -O '$dest' $url").strip("'$dest' $url").split()
for line in f.readlines():
fields = line.split()
binurl = self.binrepo + "/" + self._package + "/"
if len(fields) == 2:
checksum, source = fields
binurl += source + "/"
else:
pattern = re.compile(r'^(.*) \((.*)\) = (.*)')
match = pattern.match(line)
hashtype = match.group(1)
source = match.group(2)
checksum = match.group(3)
binurl += source + "/" + hashtype.lower() + "/"
binurl += checksum + "/" + source
cmd = downloader + [join(self._git.path, source), binurl]
f.close()
status, output = execcmd(cmd, show=True)
if status == 0:
return True
else:
raise Error("Failed downloading %s, retcode: %d err: %s\n", status,
output)
def _execVcs(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup")
if not kwargs.get("show") and args[0] not in localcmds:
args = list(args)
args.append("--non-interactive")
else:
kwargs["geterr"] = True
kwargs["cleanerr"] = True
cmdstr = self.vcs_command + " " + " ".join(args)
try:
return execcmd(cmdstr, **kwargs)
except Error, e:
msg = None
if e.args:
if "Permission denied" in e.args[0]:
msg = (
"It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"http://wiki.mandriva.com/en/Development/Docs/Contributor_Tricks#SSH_configuration"
" for more information.")
elif "authorization failed" in e.args[0]:
msg = ("Note that repsys does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error, "%s\n%s" % (e, msg)
raise
def build_rpm(build_cmd="a",
verbose=False,
rpmlint=True,
packager = "",
macros = []):
top = os.getcwdu()
topdir = "--define '_topdir %s'" % top
builddir = "--define '_builddir %s/%s'" % (top, "BUILD")
rpmdir = "--define '_rpmdir %s/%s'" % (top, "RPMS")
sourcedir = "--define '_sourcedir %s/%s'" % (top, "SOURCES")
specdir = "--define '_specdir %s/%s'" % (top, "SPECS")
srcrpmdir = "--define '_srcrpmdir %s/%s'" % (top, "SRPMS")
patchdir = "--define '_patchdir %s/%s'" % (top, "SOURCES")
build = os.path.join(top, "BUILD")
if not os.path.exists(build):
os.mkdir(build)
specsdir = os.path.join(top, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if packager:
packager = " --define 'packager %s'" % packager
defs = rpm_macros_defs(macros)
rpmbuild = config.get("helper", "rpmbuild", "rpmbuild")
begintime = time()
execcmd("LC_ALL=C %s -b%s %s %s %s %s %s %s %s %s %s %s" %
(rpmbuild, build_cmd, topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, packager, spec, defs), show=verbose)
endtime = time()
if rpmlint:
status, output = execcmd("rpm %s %s --define 'debug_package_and_restore %%{debug_package}' --define '_query_all_fmt %%{_srcrpmdir}/%%{SOURCERPM} %%{_rpmdir}/%%{_build_name_fmt}' -q --specfile %s" % (rpmdir, srcrpmdir, spec))
checklist = []
for ps in output.split("\n"):
for p in ps.split():
if p not in checklist and os.path.exists(p):
# We assume that only packages with matching filename and
# that has been built since we invoked rpmbuild belongs to
# this build and should be checked
stat = os.stat(p)
if stat.st_ctime > begintime and stat.st_ctime < endtime:
checklist.append(p)
if checklist:
execcmd("rpmlint %s" % string.join(checklist), show=True)
def build_rpm(build_cmd="a",
verbose=False,
rpmlint=True,
packager = "",
macros = []):
top = os.getcwdu()
topdir = "--define '_topdir %s'" % top
builddir = "--define '_builddir %s/%s'" % (top, "BUILD")
rpmdir = "--define '_rpmdir %s/%s'" % (top, "RPMS")
sourcedir = "--define '_sourcedir %s/%s'" % (top, "SOURCES")
specdir = "--define '_specdir %s/%s'" % (top, "SPECS")
srcrpmdir = "--define '_srcrpmdir %s/%s'" % (top, "SRPMS")
patchdir = "--define '_patchdir %s/%s'" % (top, "SOURCES")
build = os.path.join(top, "BUILD")
if not os.path.exists(build):
os.mkdir(build)
specsdir = os.path.join(top, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if packager:
packager = " --define 'packager %s'" % packager
defs = rpm_macros_defs(macros)
rpmbuild = config.get("helper", "rpmbuild", "rpmbuild")
begintime = time()
execcmd("LC_ALL=C %s -b%s %s %s %s %s %s %s %s %s %s %s" %
(rpmbuild, build_cmd, topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, packager, spec, defs), show=verbose)
endtime = time()
if rpmlint:
status, output = execcmd("rpm %s %s --define 'debug_package_and_restore %%{debug_package}' --define '_query_all_fmt %%{_srcrpmdir}/%%{SOURCERPM} %%{_rpmdir}/%%{_build_name_fmt}' -q --specfile %s" % (rpmdir, srcrpmdir, spec))
checklist = []
for ps in output.split("\n"):
for p in ps.split():
if p not in checklist and os.path.exists(p):
# We assume that only packages with matching filename and
# that has been built since we invoked rpmbuild belongs to
# this build and should be checked
stat = os.stat(p)
if stat.st_ctime > begintime and stat.st_ctime < endtime:
checklist.append(p)
if checklist:
execcmd("rpmlint %s" % string.join(checklist), show=True)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
args.append("| less")
rawcmd = " ".join(args)
execcmd(rawcmd, show=True)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
args.append("| less")
rawcmd = " ".join(args)
execcmd(rawcmd, show=True)
def _getinfo(self):
args = [
"rpm", "-qp", "--qf", "%{name} %{epoch} %{release} %{version}",
self.filename
]
status, output = execcmd(args)
self.name, self.epoch, self.release, self.version = output.split()
if self.epoch == "(none)":
self.epoch = None
def _execsvnlook(self, cmd, *args, **kwargs):
execcmd_args = ["svnlook", cmd, self.repospath]
self._add_txnrev(execcmd_args, kwargs)
execcmd_args.extend(args)
execcmd_kwargs = {}
keywords = ["show", "noerror"]
for key in keywords:
if kwargs.has_key(key):
execcmd_kwargs[key] = kwargs[key]
return execcmd(*execcmd_args, **execcmd_kwargs)
def _execVcslook(self, cmd, *args, **kwargs):
execcmd_args = ["svnlook", cmd, self.repospath]
self._add_txnrev(execcmd_args, kwargs)
execcmd_args += args
execcmd_kwargs = {}
keywords = ["show", "noerror"]
for key in keywords:
if key in kwargs:
execcmd_kwargs[key] = kwargs[key]
return execcmd(*execcmd_args, **execcmd_kwargs)
def _execVcslook(self, cmd, *args, **kwargs):
execcmd_args = [self.execcmd, cmd, self.repospath]
self._add_txnrev(execcmd_args, kwargs)
execcmd_args += args
execcmd_kwargs = {}
keywords = ["show", "noerror"]
for key in keywords:
if kwargs.has_key(key):
execcmd_kwargs[key] = kwargs[key]
return execcmd(*execcmd_args, **execcmd_kwargs)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(str(limit))
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
pager = shlex.split(os.environ.get("PAGER", "less"))
p = subprocess.Popen(args, stdout=subprocess.PIPE)
p2 = subprocess.Popen(pager, stdin=p.stdout)
p2.wait()
p.wait()
else:
execcmd(args, show=True)
def init(self, url, targetpath, fullnames=True, branch=None, **kwargs):
# verify repo url
execcmd("svn", "info", url)
topurl = dirname(url)
trunk = basename(url)
tags = "releases"
# cloning svn braches as well should rather be optionalif reenabled..
#cmd = ["svn", "init", topurl, "--trunk="+trunk, "--tags="+tags", targetpath]
cmd = ["svn", "init", url, abspath(targetpath)]
self._execVcs(*cmd, **kwargs)
environ.update({
"GIT_WORK_TREE": abspath(targetpath),
"GIT_DIR": join(abspath(targetpath), ".git")
})
if fullnames:
usermap = UserTagParser()
# store configuration in local git config so that'll be reused later when ie. updating
gitconfig = {
"svn-remote.authorlog.url": usermap.url,
"svn-remote.authorlog.defaultmail": usermap.defaultmail
}
self.configset(gitconfig)
if branch:
execcmd(("git", "init", "-q", self.path), **kwargs)
execcmd(("git", "checkout", "-q", branch), **kwargs)
cmd = ["svn", "rebase", "--local"]
status, output = self._execVcs(*cmd, **kwargs)
return True
def _execVcs(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup", "mv")
cmd = self.vcs_command + list(args)
if self.env_defaults:
self._set_env()
kwargs["collecterr"] = kwargs.get("collecterr", False)
if kwargs.get("show"):
if not kwargs.get("local"):
kwargs["collecterr"] = True
else:
if self.vcs_command is "svn" and args[0] not in localcmds:
cmd.append("--non-interactive")
else:
if args[0] == "mv":
kwargs["collecterr"] = False
kwargs["cleanerr"] = kwargs.get("cleanerr", True)
if kwargs.get("xml"):
cmd.append("--xml")
try:
if args[0] in ('info', 'checkout', 'log'):
kwargs['info'] = True
else:
kwargs['info'] = False
return execcmd(*cmd, **kwargs)
except Error as e:
msg = None
if e.args:
if "Permission denied" in e.args:
msg = ("It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"https://wiki.mageia.org/en/Packagers_ssh"
" for more information.")
elif "authorization failed" in e.args:
msg = ("Note that repsys does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error("%s\n%s" % (e, msg))
raise
def submit(urls,
target,
define=[],
submithost=None,
port=None,
atonce=False,
debug=False,
keeplog=False):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
raise Error, "no submit host defined in configuration"
if port is None:
port = config.getint("submit", "port", "22")
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", "-p", str(port), submithost, createsrpm, "-t", target]
if debug:
baseargs.append("--debug")
if keeplog:
baseargs.append("--keeplog")
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
status, output = execcmd(cmdargs, noerror=1)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.stderr.write("\n")
sys.exit(status)
def upload_binary(topdir, filename):
filepath = os.path.join(topdir, 'SOURCES', filename)
if not os.path.exists(filepath):
raise Error("'%s' was not found" % filepath)
sha1sum = file_hash(filepath)
if binary_exists(sha1sum):
return
host = config.get("binrepo", "upload_host")
upload_bin_helper = get_helper("upload-bin")
command = ["ssh", host, upload_bin_helper, filename]
try:
filein = open(filepath, 'r')
except Error as e:
raise Error("Could not open file %s\n" % filepath)
status, output = execcmd(*command,
show=True,
collecterr=True,
stdin=filein)
def patch_spec(pkgdirurl, patchfile, log=""):
#FIXME use get_spec
svn = SVN()
tmpdir = tempfile.mktemp()
try:
geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
svn.checkout(geturl, tmpdir)
speclist = glob.glob(os.path.join(tmpdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
status, output = execcmd(["patch", spec, patchfile])
if status != 0:
raise Error, "can't apply patch:\n%s\n" % output
else:
svn.commit(tmpdir, log="")
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def patch_spec(pkgdirurl, patchfile, log=""):
#FIXME use get_spec
svn = SVN()
tmpdir = tempfile.mktemp()
try:
geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
svn.checkout(geturl, tmpdir)
speclist = glob.glob(os.path.join(tmpdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
status, output = execcmd(["patch", spec, patchfile])
if status != 0:
raise Error, "can't apply patch:\n%s\n" % output
else:
svn.commit(tmpdir, log="")
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def drop_ssh_if_no_auth(self, url):
if url and url.startswith("svn+ssh://"):
cmd = [
"svn", "info", "--non-interactive", "--no-newline",
"--show-item", "url", url
]
status, output = self._execVcs(*cmd,
local=True,
noerror=True,
show=False)
if status == 1 and (("E170013" in output) or
("E210002" in output)):
url = url.replace("svn+ssh://", "svn://")
status, output = execcmd(*cmd,
local=True,
noerror=True,
show=False)
if status == 0 and output == url:
pass
return url
def submit(urls, target, define=[], submithost=None, port=None,
atonce=False, debug=False, keeplog=False):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
raise Error, "no submit host defined in configuration"
if port is None:
port = config.getint("submit", "port", "22")
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", "-p", str(port), submithost, createsrpm,
"-t", target]
if debug:
baseargs.append("--debug")
if keeplog:
baseargs.append("--keeplog")
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
status, output = execcmd(cmdargs, noerror=1)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.stderr.write("\n")
sys.exit(status)
def download_binary(topdir, sha1, filename):
fmt = config.get("global", "download-command", "wget -c -O '$dest' $url")
url = config.get("binrepo", "download_url", "http://binrepo.mageia.org/")
url = mirror.normalize_path(url + "/" + sha1)
dest = os.path.join(topdir, 'SOURCES', filename)
if os.path.exists(dest):
if file_hash(dest) == sha1:
return 1
else:
raise Error("File with incorrect sha1sum: %s" % dest)
context = {"dest": dest, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError as e:
raise Error("invalid variable %r in download-command "\
"configuration option" % e)
try:
status, output = execcmd(cmd, show=True)
except Error as e:
os.unlink(dest)
raise Error("Could not download file %s\n" % url)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
# extract the submit host from the svn host
type, rest = urllib.splittype(pkgdirurl)
host, path = urllib.splithost(rest)
user, host = urllib.splituser(host)
submithost, port = urllib.splitport(host)
del type, user, port, path, rest
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", submithost, createsrpm, "-t", target]
if not sid:
sid = uuid.uuid4()
define.append("sid=%s" % sid)
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
command = subprocess.list2cmdline(cmdargs)
status, output = execcmd(command)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.exit(status)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
# extract the submit host from the svn host
type, rest = urllib.splittype(pkgdirurl)
host, path = urllib.splithost(rest)
user, host = urllib.splituser(host)
submithost, port = urllib.splitport(host)
del type, user, port, path, rest
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", submithost, createsrpm, "-t", target]
if not sid:
sid = uuid.uuid4()
define.append("sid=%s" % sid)
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side repsys versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
command = subprocess.list2cmdline(cmdargs)
status, output = execcmd(command)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.exit(status)
def unpack(self, topdir):
args = ["rpm", "-i", "--define", "_topdir %s" % (topdir),
self.filename]
execcmd(args)
def maintdb(maintdb_args):
host = config.get("maintdb", "host", "maintdb.mageia.org")
maintdb_helper = get_helper("maintdb")
command = ["ssh", host, maintdb_helper] + maintdb_args
execcmd(command, show=True)
sys.exit(0)
def get_srpm(pkgdirurl,
mode = "current",
targetdirs = None,
version = None,
release = None,
revision = None,
packager = "",
revname = 0,
svnlog = 0,
scripts = [],
submit = False,
template = None,
distro = None,
macros = [],
verbose = 0,
strict = False):
svn = SVN()
tmpdir = tempfile.mktemp()
topdir = "_topdir %s" % tmpdir
builddir = "_builddir %s/%s" % (tmpdir, "BUILD")
rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS")
sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES")
specdir = "_specdir %s/%s" % (tmpdir, "SPECS")
srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS")
patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES")
temppath = "_tmppath %s" % (tmpdir)
rpmdefs = [("--define", expr) for expr in (topdir, builddir, rpmdir,
sourcedir, specdir, srcrpmdir, patchdir, temppath)]
try:
if mode == "version":
geturl = layout.checkout_url(pkgdirurl, version=version,
release=release)
elif mode == "pristine":
geturl = layout.checkout_url(pkgdirurl, pristine=True)
elif mode == "current" or mode == "revision":
#FIXME we should handle revisions specified using @REV
geturl = layout.checkout_url(pkgdirurl)
else:
raise Error, "unsupported get_srpm mode: %s" % mode
strict = strict or config.getbool("submit", "strict-revision", False)
if strict and not rev_touched_url(geturl, revision):
#FIXME would be nice to have the revision number even when
# revision is None
raise Error, "the revision %s does not change anything "\
"inside %s" % (revision or "HEAD", geturl)
mirror.info(geturl)
svn.export(geturl, tmpdir, rev=revision)
srpmsdir = os.path.join(tmpdir, "SRPMS")
os.mkdir(srpmsdir)
specsdir = os.path.join(tmpdir, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if config.getbool("srpm", "run-prep", False):
makefile = os.path.join(tmpdir, "Makefile")
if os.path.exists(makefile):
execcmd(("make", "-C", tmpdir, "srpm-prep"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if svnlog:
submit = not not revision
log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
template=template, macros=macros, exported=tmpdir)
for script in scripts:
#FIXME revision can be "None"
status, output = execcmd(script, tmpdir, spec, str(revision),
noerror=1)
if status != 0:
raise Error, "script %s failed" % script
if packager:
packager = " --define 'packager %s'" % packager
if distro:
cmpdistro = distro.lower()
for target in cgiutil.get_targets():
if target.name.lower() == cmpdistro:
macros.extend(target.macros)
break
else:
raise Error, "no such submit target in configuration: %s" % (distro)
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
args = [sourcecmd, "-bs", "--nodeps"]
for pair in rpmdefs:
args.extend(pair)
for pair in macros:
args.extend(("--define", "%s %s" % pair))
args.append(spec)
try:
execcmd(args)
except CommandError, e:
if config.getbool("global", "verbose"):
cmdline = e.cmdline + "\n"
else:
cmdline = ""
raise Error, ("error while creating the source RPM "
"(with %s):\n%s%s" % (sourcecmd, cmdline, e.output))
# copy the generated SRPMs to their target locations
targetsrpms = []
urlrev = None
if revname:
urlrev = revision or layout.get_url_revision(geturl)
if not targetdirs:
targetdirs = (".",)
srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
if not srpms:
# something fishy happened
raise Error, "no SRPMS were found at %s" % srpmsdir
for srpm in srpms:
name = os.path.basename(srpm)
if revname:
name = "@%s:%s" % (urlrev, name)
for targetdir in targetdirs:
newpath = os.path.join(targetdir, name)
targetsrpms.append(newpath)
if os.path.exists(newpath):
# should we warn?
os.unlink(newpath)
shutil.copy(srpm, newpath)
if verbose:
sys.stderr.write("Wrote: %s\n" % newpath)
return targetsrpms
def build_rpm(build_cmd="b",
verbose=True,
rpmlint=True,
short_circuit=False,
packager=None,
installdeps=True,
use_dnf=False,
svnlog=False,
fullnames=True,
macros=[],
**rpmargs):
top = os.getcwd()
vcs = detectVCS(top)
topdir = "_topdir %s" % top
builddir = "_builddir %s/%s" % (top, "BUILD")
rpmdir = "_rpmdir %s/%s" % (top, "RPMS")
srcrpmdir = "_srcrpmdir %s/%s" % (top, "SRPMS")
specsdir = join(top, "SPECS")
# fedora & abf layout
if vcs.vcs_name == "git" and not os.path.exists(specsdir):
sourcedir = "_sourcedir " + top
specdir = "_specdir " + top
patchdir = "_patchdir " + top
specsdir = top
else:
# mageia layout
sourcedir = "_sourcedir %s/%s" % (top, "SOURCES")
specdir = "_specdir %s/%s" % (top, "SPECS")
patchdir = "_patchdir %s/%s" % (top, "SOURCES")
build = join(top, "BUILD")
if not os.path.exists(build):
os.mkdir(build)
speclist = glob.glob(join(specsdir, "*.spec"))
if not speclist:
raise Error("no spec files found")
spec = speclist[0]
# If we're building package with %changelog, we'll make a temporary
# copy of the spec file with %changelog applied that we'll use for
# building. This way we avoid modifying files in repository.
# TODO: implement support for external changelog in rpm
if svnlog:
specsdir = tempfile.mkdtemp()
shutil.copy(spec, specsdir)
specdir = "_specdir " + specsdir
spec = os.path.join(specsdir, os.path.basename(spec))
info = vcs.info2(top)
pkgdirurl = layout.remove_current(info["URL"])
log.specfile_svn2rpm(pkgdirurl,
spec,
rev=None,
submit=False,
template=None,
macros=macros,
exported=top,
fullnames=fullnames)
rpmdefs = [("--define", expr)
for expr in (topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir)]
if packager:
rpmdefs.append(("--define", "packager %s" % packager))
if rpmlint:
rpmdefs.append(("--define", "_build_pkgcheck_set %{_bindir}/rpmlint"))
rpmbuild = config.get("helper", "rpmbuild", "rpmbuild")
args = [rpmbuild, spec]
if short_circuit:
args.append("--short-circuit")
for pair in rpmdefs:
args.extend(pair)
for pair in macros:
args.extend(("--define", "%s %s" % pair))
args.extend(("--define", "_disable_source_fetch 0"))
args.extend(*rpmargs.values())
os.environ["LC_ALL"] = "C"
# First check whether dependencies are satisfied
status, output = execcmd(*args + ["--nobuild"],
show=verbose,
collecterr=True,
noerror=True)
if status:
if "error: Failed build dependencies:" in output:
if not installdeps:
raise Error("Automatic installation of dependencies disabled,"
"aborting...")
else:
if verbose:
print("Installing missing build dependencies")
if use_dnf:
pkg_mgr_base = ["dnf"]
pkg_mgr_builddep = pkg_mgr_base + [
"--assume-yes", "--setopt=install_weak_deps=False",
"builddep"
]
else:
pkg_mgr_base = ["urpmi"]
pkg_mgr_builddep = pkg_mgr_base + [
"--auto", "--buildrequires", "--no-recommends"
]
if os.getuid() != 0:
print(
"Trying to obtain privileges for installing build dependencies:"
)
sudocheck = ["sudo", "-l"] + pkg_mgr_base
status, output = execcmd(*sudocheck,
collecter=True,
noerror=True)
if status:
raise Error(
"%s\nFailed! Cannot proceed without, aborting..." %
output.splitlines()[-1])
cmd_base = ["sudo"] + pkg_mgr_builddep
else:
cmd_base = pkg_mgr_builddep
cmd = cmd_base + [spec]
status, output = execcmd(*cmd,
show=verbose,
collecter=True,
noerror=True)
status, output = execcmd(*args + ["-b" + build_cmd], show=verbose)
if svnlog:
if os.path.isdir(specsdir):
shutil.rmtree(specsdir)
if pst:
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
sys.stderr.write("warning: %s not found, skipping\n" % sourcepath)
elif download and not os.path.isfile(sourcepath):
print "%s not found, downloading from %s" % (sourcepath, url)
fmt = config.get("global", "download-command",
"wget -c -O '$dest' $url")
context = {"dest": sourcepath, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError, e:
raise Error, "invalid variable %r in download-command "\
"configuration option" % e
execcmd(cmd, show=True)
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
raise Error, "file not found: %s" % sourcepath
# rm entries not found in sources and still in svn
found = os.listdir(sourcesdir)
toremove = []
for entry in found:
if entry == ".svn":
continue
status = sourcesst.get(entry)
if status is None and entry not in sources:
path = os.path.join(sourcesdir, entry)
toremove.append(path)
for path in toremove:
def update(self, targetpath, clone=False, **kwargs):
environ.update({
"GIT_WORK_TREE": abspath(targetpath),
"GIT_DIR": join(abspath(targetpath), ".git")
})
if not clone:
cmd = ["svn", "log", "--oneline", "--limit=1"]
retval, result = self._execVcs(*cmd)
if retval:
return retval
revision = result.split()
if revision[0][0] == 'r':
startrev = "-r" + str(int(revision[0][1:]) + 1)
else:
startrev = "BASE"
cmd = ["svn", "propget", "svn:entry:committed-rev"]
retval, lastrev = self._execVcs(*cmd)
if retval:
return retval
#cmd = ["config", "--get-regexp", '^svn-remote.svn.(url|fetch)']
cmd = ["config", "--get", "svn-remote.svn.url"]
retval, result = self._execVcs(*cmd)
if retval:
return retval
#result = result.strip().split()
#url = result[1] + "/" + result[3].split(":")[0]
url = result.strip()
# To speed things up on huge repositories, we'll just grab all the
# revision numbers for this specific directory and grab these only
# in stead of having to go through each and every revision...
cmd = ["svn", "log", "-g", "--xml", url]
if not clone:
cmd.append("%s:%s" % (startrev, lastrev))
retval, result = execcmd(*cmd)
if retval:
return retval
xmllog = ElementTree.fromstring(result)
logentries = xmllog.getiterator("logentry")
revisions = []
for entry in logentries:
revisions.append(int(entry.attrib["revision"]))
revisions.sort()
fetchcmd = ["svn", "fetch", "--log-window-size=1000"]
gitconfig = self.configget("svn-remote.authorlog")
if gitconfig:
usermap = UserTagParser(
url=gitconfig.get("svn-remote.authorlog.url"),
defaultmail=gitconfig.get("svn-remote.authorlog.defaultmail"))
usermapfile = usermap.get_user_map_file()
fetchcmd.extend(("--authors-file", usermapfile))
fetchcmd.append("")
while revisions:
fetchcmd[-1] = "-r%d" % revisions.pop(0)
self._execVcs(*fetchcmd, **kwargs)
if gitconfig:
usermap.cleanup()
cmd = [
"svn", "rebase", "--log-window-size=1000", "--local",
"--fetch-all", "git-svn"
]
status, output = self._execVcs(*cmd, **kwargs)
if status == 0:
return [x.split() for x in output.split()]
return None
def sync(dryrun=False, commit=False, download=False):
topdir = getpkgtopdir()
svn = detectVCS(topdir)
spath = binrepo.sources_path(topdir)
binrepoentries = binrepo.parse_sources(spath)
# run svn info because svn st does not complain when topdir is not an
# working copy
svn.info(topdir)
specsdir = os.path.join(topdir, "SPECS/")
sourcesdir = os.path.join(topdir, "SOURCES/")
for path in (specsdir, sourcesdir):
if not os.path.isdir(path):
raise Error("%s directory not found" % path)
specs = glob.glob(os.path.join(specsdir, "*.spec"))
if not specs:
raise Error("no .spec files found in %s" % specsdir)
specpath = specs[0] # FIXME better way?
try:
rpm.addMacro("_topdir", os.path.abspath(topdir))
spec = rpm.TransactionSet().parseSpec(specpath)
except rpm.error as e:
raise Error("could not load spec file: %s" % e)
srclist = spec.sources if isinstance(spec.sources, (list, tuple)) \
else spec.sources()
sources = dict(
(os.path.basename(name), name) for name, no, flags in srclist)
sourcesst = dict((os.path.basename(path), (path, st))
for st, path in svn.status(sourcesdir, noignore=True))
toadd = []
toremove = []
# add the spec file itself, in case of a new package
specstl = svn.status(specpath, noignore=True)
if specstl:
specst, _ = specstl[0]
if specst == "?":
toadd.append(specpath)
# add source files:
for source, url in sources.items():
sourcepath = os.path.join(sourcesdir, source)
if sourcesst.get(source):
if not (source
in binrepoentries) and sourcesst.get(source)[1] == '?':
toadd.append(sourcepath)
elif not download and not os.path.isfile(sourcepath):
sys.stderr.write("warning: %s not found\n" % sourcepath)
elif download and not os.path.isfile(sourcepath):
print("%s not found, downloading from %s" % (sourcepath, url))
fmt = config.get("global", "download-command",
"wget -c -O '$dest' $url")
context = {"dest": sourcepath, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError as e:
raise Error("invalid variable %r in download-command "\
"configuration option" % e)
try:
status, output = execcmd(cmd, show=True)
except Error as e:
os.unlink(sourcepath)
raise Error("Could not download file %s\n" % url)
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
raise Error("file not found: %s" % sourcepath)
# rm entries not found in sources and still in svn
found = os.listdir(sourcesdir)
for entry in found:
if entry == ".svn" or entry == binrepo.SOURCES_FILE:
continue
status = sourcesst.get(entry)
path = os.path.join(sourcesdir, entry)
if entry not in sources:
if status is None or entry in binrepoentries:
toremove.append(path)
for path in toremove:
print("D\t%s" % path)
if not dryrun:
delete([path], commit=commit)
for path in toadd:
print("A\t%s" % path)
if not dryrun:
upload([path], commit=commit)
def get_srpm(pkgdirurl,
mode="current",
targetdirs=None,
version=None,
release=None,
revision=None,
packager="",
revname=0,
svnlog=0,
scripts=[],
submit=False,
template=None,
macros=[],
verbose=0,
strict=False,
fullnames=False):
svn = detectVCS(pkgdirurl)
tmpdir = tempfile.mktemp()
topdir = "_topdir %s" % tmpdir
builddir = "_builddir %s/%s" % (tmpdir, "BUILD")
rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS")
sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES")
specdir = "_specdir %s/%s" % (tmpdir, "SPECS")
srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS")
patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES")
temppath = "_tmppath %s" % (tmpdir)
rpmdefs = [("--define", expr)
for expr in (topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, temppath)]
try:
if mode == "version":
geturl = layout.checkout_url(pkgdirurl,
version=version,
release=release)
elif mode == "pristine":
geturl = layout.checkout_url(pkgdirurl, pristine=True)
elif mode == "current" or mode == "revision":
#FIXME we should handle revisions specified using @REV
geturl = layout.checkout_url(pkgdirurl)
else:
raise Error("unsupported get_srpm mode: %s" % mode)
strict = strict or config.getbool("submit", "strict-revision", False)
if strict and not rev_touched_url(geturl, revision):
#FIXME would be nice to have the revision number even when
# revision is None
raise Error("the revision %s does not change anything "\
"inside %s" % (revision or "HEAD", geturl))
mirror.info(geturl)
svn.export(geturl, tmpdir, rev=revision)
binrepo.download_binaries(tmpdir)
srpmsdir = os.path.join(tmpdir, "SRPMS")
os.mkdir(srpmsdir)
specsdir = os.path.join(tmpdir, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if config.getbool("srpm", "run-prep", False):
makefile = os.path.join(tmpdir, "Makefile")
if os.path.exists(makefile):
execcmd(("make", "-C", tmpdir, "srpm-prep"))
if not speclist:
raise Error("no spec files found")
spec = speclist[0]
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
if packager:
packager = " --define 'packager %s'" % packager
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
args = [sourcecmd, "-bs", "--nodeps"]
for pair in rpmdefs:
args.extend(pair)
for pair in macros:
args.extend(("--define", "%s %s" % pair))
args.append(spec)
if svnlog:
submit = not not revision
try:
log.specfile_svn2rpm(pkgdirurl,
spec,
revision,
submit=submit,
template=template,
macros=macros,
exported=tmpdir,
fullnames=fullnames)
except:
#cmd = [sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir
execcmd(args)
cp_srpms(revision, revname, geturl, targetdirs, srpmsdir,
verbose)
log.specfile_svn2rpm(pkgdirurl,
spec,
revision,
submit=submit,
template=template,
macros=macros,
exported=tmpdir,
create=True)
for script in scripts:
#FIXME revision can be "None"
status, output = execcmd(script,
tmpdir,
spec,
str(revision),
noerror=1)
if status != 0:
raise Error("script %s failed" % script)
try:
execcmd(args)
except CommandError as e:
if config.getbool("global", "verbose"):
cmdline = e.cmdline + "\n"
else:
cmdline = ""
raise Error("error while creating the source RPM "
"(with %s):\n%s%s" % (sourcecmd, cmdline, e.output))
# copy the generated SRPMs to their target locations
targetsrpms = cp_srpms(revision, revname, geturl, targetdirs, srpmsdir,
verbose)
return targetsrpms
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def put_srpm(srpmfile,
markrelease=False,
striplog=True,
branch=None,
baseurl=None,
baseold=None,
logmsg=None,
rename=True):
svn = SVN()
srpm = SRPM(srpmfile)
tmpdir = tempfile.mktemp()
if baseurl:
pkgurl = mirror._joinurl(baseurl, srpm.name)
else:
pkgurl = layout.package_url(srpm.name, distro=branch, mirrored=False)
print "Importing package to %s" % pkgurl
try:
if srpm.epoch:
version = "%s:%s" % (srpm.epoch, srpm.version)
else:
version = srpm.version
versionurl = "/".join([pkgurl, "releases", version])
releaseurl = "/".join([versionurl, srpm.release])
currenturl = "/".join([pkgurl, "current"])
currentdir = os.path.join(tmpdir, "current")
#FIXME when pre-commit hook fails, there's no clear way to know
# what happened
ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory")
if ret or not svn.ls(currenturl, noerror=1):
svn.checkout(pkgurl, tmpdir)
svn.mkdir(os.path.join(tmpdir, "releases"))
svn.mkdir(currentdir)
svn.mkdir(os.path.join(currentdir, "SPECS"))
svn.mkdir(os.path.join(currentdir, "SOURCES"))
#svn.commit(tmpdir,log="Created package structure.")
version_exists = 1
else:
if svn.ls(releaseurl, noerror=1):
raise Error, "release already exists"
svn.checkout("/".join([pkgurl, "current"]), tmpdir)
svn.mkdir(versionurl,
noerror=1,
log="Created directory for version %s." % version)
currentdir = tmpdir
specsdir = os.path.join(currentdir, "SPECS")
sourcesdir = os.path.join(currentdir, "SOURCES")
unpackdir = tempfile.mktemp()
os.mkdir(unpackdir)
try:
srpm.unpack(unpackdir)
uspecsdir = os.path.join(unpackdir, "SPECS")
usourcesdir = os.path.join(unpackdir, "SOURCES")
uspecsentries = os.listdir(uspecsdir)
usourcesentries = os.listdir(usourcesdir)
specsentries = os.listdir(specsdir)
sourcesentries = os.listdir(sourcesdir)
# Remove old entries
for entry in [x for x in specsentries if x not in uspecsentries]:
if entry == ".svn":
continue
entrypath = os.path.join(specsdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
for entry in [
x for x in sourcesentries if x not in usourcesentries
]:
if entry == ".svn":
continue
entrypath = os.path.join(sourcesdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
# Copy all files
execcmd(["cp", "-rf", uspecsdir, currentdir])
execcmd(["cp", "-rf", usourcesdir, currentdir])
# Add new entries
for entry in [x for x in uspecsentries if x not in specsentries]:
entrypath = os.path.join(specsdir, entry)
svn.add(entrypath)
for entry in [
x for x in usourcesentries if x not in sourcesentries
]:
entrypath = os.path.join(sourcesdir, entry)
svn.add(entrypath)
finally:
if os.path.isdir(unpackdir):
shutil.rmtree(unpackdir)
specs = glob.glob(os.path.join(specsdir, "*.spec"))
if not specs:
raise Error, "no spec file found on %s" % specsdir
if len(specs) > 1:
raise Error, "more than one spec file found on %s" % specsdir
specpath = specs[0]
if rename:
specfile = os.path.basename(specpath)
specname = specfile[:-len(".spec")]
if specname != srpm.name:
newname = srpm.name + ".spec"
newpath = os.path.join(specsdir, newname)
sys.stderr.write("warning: renaming spec file to '%s' "
"(use -n to disable it)\n" % (newname))
os.rename(specpath, newpath)
try:
svn.remove(specpath)
except Error:
# file not tracked
svn.revert(specpath)
svn.add(newpath)
specpath = newpath
if striplog:
specpath = specpath
fspec = open(specpath)
spec, chlog = log.split_spec_changelog(fspec)
fspec.close()
fspec = open(specpath, "w")
fspec.writelines(spec)
fspec.close()
chlog.seek(0, os.SEEK_END)
if chlog.tell() != 0:
chlog.seek(0)
#FIXME move it to layout.py
oldurl = baseold or config.get("log", "oldurl")
pkgoldurl = mirror._joinurl(oldurl, srpm.name)
svn.mkdir(pkgoldurl,
noerror=1,
log="created old log directory for %s" % srpm.name)
logtmp = tempfile.mktemp()
try:
svn.checkout(pkgoldurl, logtmp)
miscpath = os.path.join(logtmp, "log")
fmisc = open(miscpath, "w+")
fmisc.writelines(chlog)
fmisc.close()
svn.add(miscpath)
svn.commit(logtmp,
log="imported old log for %s" % srpm.name)
finally:
if os.path.isdir(logtmp):
shutil.rmtree(logtmp)
svn.commit(tmpdir, log=logmsg or ("imported package %s" % srpm.name))
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
# Do revision and pristine tag copies
pristineurl = layout.checkout_url(pkgurl, pristine=True)
svn.remove(pristineurl,
noerror=1,
log="Removing previous pristine/ directory.")
currenturl = layout.checkout_url(pkgurl)
svn.copy(currenturl,
pristineurl,
log="Copying release %s-%s to pristine/ directory." %
(version, srpm.release))
if markrelease:
svn.copy(currenturl,
releaseurl,
log="Copying release %s-%s to releases/ directory." %
(version, srpm.release))
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
sys.stderr.write("warning: %s not found, skipping\n" %
sourcepath)
elif download and not os.path.isfile(sourcepath):
print "%s not found, downloading from %s" % (sourcepath, url)
fmt = config.get("global", "download-command",
"wget -c -O '$dest' $url")
context = {"dest": sourcepath, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError, e:
raise Error, "invalid variable %r in download-command "\
"configuration option" % e
execcmd(cmd, show=True)
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
raise Error, "file not found: %s" % sourcepath
# rm entries not found in sources and still in svn
found = os.listdir(sourcesdir)
toremove = []
for entry in found:
if entry == ".svn":
continue
status = sourcesst.get(entry)
if status is None and entry not in sources:
path = os.path.join(sourcesdir, entry)
toremove.append(path)
for path in toremove:
def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
baseurl=None, baseold=None, logmsg=None, rename=True):
svn = SVN()
srpm = SRPM(srpmfile)
tmpdir = tempfile.mktemp()
if baseurl:
pkgurl = mirror._joinurl(baseurl, srpm.name)
else:
pkgurl = layout.package_url(srpm.name, distro=branch,
mirrored=False)
print "Importing package to %s" % pkgurl
try:
if srpm.epoch:
version = "%s:%s" % (srpm.epoch, srpm.version)
else:
version = srpm.version
versionurl = "/".join([pkgurl, "releases", version])
releaseurl = "/".join([versionurl, srpm.release])
currenturl = "/".join([pkgurl, "current"])
currentdir = os.path.join(tmpdir, "current")
#FIXME when pre-commit hook fails, there's no clear way to know
# what happened
ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory")
if ret or not svn.ls(currenturl, noerror=1):
svn.checkout(pkgurl, tmpdir)
svn.mkdir(os.path.join(tmpdir, "releases"))
svn.mkdir(currentdir)
svn.mkdir(os.path.join(currentdir, "SPECS"))
svn.mkdir(os.path.join(currentdir, "SOURCES"))
#svn.commit(tmpdir,log="Created package structure.")
version_exists = 1
else:
if svn.ls(releaseurl, noerror=1):
raise Error, "release already exists"
svn.checkout("/".join([pkgurl, "current"]), tmpdir)
svn.mkdir(versionurl, noerror=1,
log="Created directory for version %s." % version)
currentdir = tmpdir
specsdir = os.path.join(currentdir, "SPECS")
sourcesdir = os.path.join(currentdir, "SOURCES")
unpackdir = tempfile.mktemp()
os.mkdir(unpackdir)
try:
srpm.unpack(unpackdir)
uspecsdir = os.path.join(unpackdir, "SPECS")
usourcesdir = os.path.join(unpackdir, "SOURCES")
uspecsentries = os.listdir(uspecsdir)
usourcesentries = os.listdir(usourcesdir)
specsentries = os.listdir(specsdir)
sourcesentries = os.listdir(sourcesdir)
# Remove old entries
for entry in [x for x in specsentries
if x not in uspecsentries]:
if entry == ".svn":
continue
entrypath = os.path.join(specsdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
for entry in [x for x in sourcesentries
if x not in usourcesentries]:
if entry == ".svn":
continue
entrypath = os.path.join(sourcesdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
# Copy all files
execcmd(["cp", "-rf", uspecsdir, currentdir])
execcmd(["cp", "-rf", usourcesdir, currentdir])
# Add new entries
for entry in [x for x in uspecsentries
if x not in specsentries]:
entrypath = os.path.join(specsdir, entry)
svn.add(entrypath)
for entry in [x for x in usourcesentries
if x not in sourcesentries]:
entrypath = os.path.join(sourcesdir, entry)
svn.add(entrypath)
finally:
if os.path.isdir(unpackdir):
shutil.rmtree(unpackdir)
specs = glob.glob(os.path.join(specsdir, "*.spec"))
if not specs:
raise Error, "no spec file found on %s" % specsdir
if len(specs) > 1:
raise Error, "more than one spec file found on %s" % specsdir
specpath = specs[0]
if rename:
specfile = os.path.basename(specpath)
specname = specfile[:-len(".spec")]
if specname != srpm.name:
newname = srpm.name + ".spec"
newpath = os.path.join(specsdir, newname)
sys.stderr.write("warning: renaming spec file to '%s' "
"(use -n to disable it)\n" % (newname))
os.rename(specpath, newpath)
try:
svn.remove(specpath)
except Error:
# file not tracked
svn.revert(specpath)
svn.add(newpath)
specpath = newpath
if striplog:
specpath = specpath
fspec = open(specpath)
spec, chlog = log.split_spec_changelog(fspec)
fspec.close()
fspec = open(specpath, "w")
fspec.writelines(spec)
fspec.close()
chlog.seek(0, os.SEEK_END)
if chlog.tell() != 0:
chlog.seek(0)
#FIXME move it to layout.py
oldurl = baseold or config.get("log", "oldurl")
pkgoldurl = mirror._joinurl(oldurl, srpm.name)
svn.mkdir(pkgoldurl, noerror=1,
log="created old log directory for %s" % srpm.name)
logtmp = tempfile.mktemp()
try:
svn.checkout(pkgoldurl, logtmp)
miscpath = os.path.join(logtmp, "log")
fmisc = open(miscpath, "w+")
fmisc.writelines(chlog)
fmisc.close()
svn.add(miscpath)
svn.commit(logtmp,
log="imported old log for %s" % srpm.name)
finally:
if os.path.isdir(logtmp):
shutil.rmtree(logtmp)
svn.commit(tmpdir,
log=logmsg or ("imported package %s" % srpm.name))
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
# Do revision and pristine tag copies
pristineurl = layout.checkout_url(pkgurl, pristine=True)
svn.remove(pristineurl, noerror=1,
log="Removing previous pristine/ directory.")
currenturl = layout.checkout_url(pkgurl)
svn.copy(currenturl, pristineurl,
log="Copying release %s-%s to pristine/ directory." %
(version, srpm.release))
if markrelease:
svn.copy(currenturl, releaseurl,
log="Copying release %s-%s to releases/ directory." %
(version, srpm.release))
def unpack(self, topdir):
args = [
"rpm", "-i", "--define",
"_topdir %s" % (topdir), self.filename
]
execcmd(args)
def get_srpm(pkgdirurl,
mode="current",
targetdirs=None,
version=None,
release=None,
revision=None,
packager="",
revname=0,
svnlog=0,
scripts=[],
submit=False,
template=None,
distro=None,
macros=[],
verbose=0,
strict=False):
svn = SVN()
tmpdir = tempfile.mktemp()
topdir = "_topdir %s" % tmpdir
builddir = "_builddir %s/%s" % (tmpdir, "BUILD")
rpmdir = "_rpmdir %s/%s" % (tmpdir, "RPMS")
sourcedir = "_sourcedir %s/%s" % (tmpdir, "SOURCES")
specdir = "_specdir %s/%s" % (tmpdir, "SPECS")
srcrpmdir = "_srcrpmdir %s/%s" % (tmpdir, "SRPMS")
patchdir = "_patchdir %s/%s" % (tmpdir, "SOURCES")
temppath = "_tmppath %s" % (tmpdir)
rpmdefs = [("--define", expr)
for expr in (topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, temppath)]
try:
if mode == "version":
geturl = layout.checkout_url(pkgdirurl,
version=version,
release=release)
elif mode == "pristine":
geturl = layout.checkout_url(pkgdirurl, pristine=True)
elif mode == "current" or mode == "revision":
#FIXME we should handle revisions specified using @REV
geturl = layout.checkout_url(pkgdirurl)
else:
raise Error, "unsupported get_srpm mode: %s" % mode
strict = strict or config.getbool("submit", "strict-revision", False)
if strict and not rev_touched_url(geturl, revision):
#FIXME would be nice to have the revision number even when
# revision is None
raise Error, "the revision %s does not change anything "\
"inside %s" % (revision or "HEAD", geturl)
mirror.info(geturl)
svn.export(geturl, tmpdir, rev=revision)
srpmsdir = os.path.join(tmpdir, "SRPMS")
os.mkdir(srpmsdir)
specsdir = os.path.join(tmpdir, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if config.getbool("srpm", "run-prep", False):
makefile = os.path.join(tmpdir, "Makefile")
if os.path.exists(makefile):
execcmd(("make", "-C", tmpdir, "srpm-prep"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if svnlog:
submit = not not revision
log.specfile_svn2rpm(pkgdirurl,
spec,
revision,
submit=submit,
template=template,
macros=macros,
exported=tmpdir)
for script in scripts:
#FIXME revision can be "None"
status, output = execcmd(script,
tmpdir,
spec,
str(revision),
noerror=1)
if status != 0:
raise Error, "script %s failed" % script
if packager:
packager = " --define 'packager %s'" % packager
if distro:
cmpdistro = distro.lower()
for target in cgiutil.get_targets():
if target.name.lower() == cmpdistro:
macros.extend(target.macros)
break
else:
raise Error, "no such submit target in configuration: %s" % (
distro)
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
args = [sourcecmd, "-bs", "--nodeps"]
for pair in rpmdefs:
args.extend(pair)
for pair in macros:
args.extend(("--define", "%s %s" % pair))
args.append(spec)
try:
execcmd(args)
except CommandError, e:
if config.getbool("global", "verbose"):
cmdline = e.cmdline + "\n"
else:
cmdline = ""
raise Error, ("error while creating the source RPM "
"(with %s):\n%s%s" % (sourcecmd, cmdline, e.output))
# copy the generated SRPMs to their target locations
targetsrpms = []
urlrev = None
if revname:
urlrev = revision or layout.get_url_revision(geturl)
if not targetdirs:
targetdirs = (".", )
srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
if not srpms:
# something fishy happened
raise Error, "no SRPMS were found at %s" % srpmsdir
for srpm in srpms:
name = os.path.basename(srpm)
if revname:
name = "@%s:%s" % (urlrev, name)
for targetdir in targetdirs:
newpath = os.path.join(targetdir, name)
targetsrpms.append(newpath)
if os.path.exists(newpath):
# should we warn?
os.unlink(newpath)
shutil.copy(srpm, newpath)
if verbose:
sys.stderr.write("Wrote: %s\n" % newpath)
return targetsrpms
def getrelease(pkgdirurl, rev=None, macros=[], exported=None, create=False):
"""Tries to obtain the version-release of the package for a
yet-not-markrelease revision of the package.
Is here where things should be changed if "automatic release increasing"
will be used.
"""
svn = SVN()
pkgcurrenturl = os.path.join(pkgdirurl, "current")
specurl = os.path.join(pkgcurrenturl, "SPECS")
srpmurl = os.path.join(pkgcurrenturl, "SRPMS")
if not create:
if exported is None:
tmpdir = tempfile.mktemp()
svn.export(specurl, tmpdir, rev=rev)
else:
tmpdir = os.path.join(exported, "SPECS")
try:
found = glob.glob(os.path.join(tmpdir, "*.spec"))
if not found:
raise Error("no .spec file found inside %s" % specurl)
specpath = found[0]
options = [("--define", expr) for expr in macros]
command = [
"rpm", "-q", "--qf", "%{EPOCH}:%{VERSION}-%{RELEASE}\n",
"--specfile", specpath
]
command.extend(options)
status, output = execcmd(*command)
releases = output.split()
try:
epoch, vr = releases[0].split(":", 1)
version, release = vr.split("-", 1)
except ValueError:
raise Error("Invalid command output: %s: %s" % \
(command, output))
#XXX check if this is the right way:
if epoch == "(none)":
ev = version
else:
ev = epoch + ":" + version
return ev, release
finally:
if exported is None and os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
else:
if exported is None:
tmpdir = tempfile.mktemp()
svn.export(specurl, tmpdir, rev=rev)
else:
tmpdir = os.path.join(exported, "SRPMS")
try:
found = glob.glob(os.path.join(srpmurl, "*.src.rpm"))
if not found:
raise Error("no .src.rpm file found inside %s" % srpmurl)
srpmpath = found[0]
options = [("--define", expr) for expr in macros]
command = [
"rpm", "-q", "--qf", "%{EPOCH}:%{VERSION}-%{RELEASE}\n",
"--specfile", specpath
]
command.extend(options)
status, output = execcmd(*command)
releases = output.split()
try:
epoch, vr = releases[0].split(":", 1)
version, release = vr.split("-", 1)
except ValueError:
raise Error("Invalid command output: %s: %s" % \
(command, output))
#XXX check if this is the right way:
if epoch == "(none)":
ev = version
else:
ev = epoch + ":" + version
return ev, release
finally:
if exported is None and os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
