def maintdb(maintdb_args):
host = config.get("maintdb", "host", "maintdb.mageia.org")
maintdb_helper = get_helper("maintdb")
cmd_args = ' '.join(maintdb_args)
command = "ssh %s %s %s" % (host, maintdb_helper, cmd_args)
execcmd(command, show=True)
sys.exit(0)
def list_targets(option, opt, val, parser):
host = config.get("submit", "host")
if host is None:
raise Error, "no submit host defined in mgarepo.conf"
createsrpm = get_helper("create-srpm")
#TODO make it configurable
command = "ssh %s %s --list" % (host, createsrpm)
execcmd(command, show=True)
sys.exit(0)
def svn_log(pkgdirurl, verbose=False, limit=None, revision=None):
mirror.info(pkgdirurl)
url = checkout_url(pkgdirurl)
svncmd = config.get("global", "svn-command", "svn")
args = [svncmd, "log", url]
if verbose:
args.append("-v")
if limit:
args.append("-l")
args.append(limit)
if revision:
args.append("-r")
args.append(revision)
if os.isatty(sys.stdin.fileno()):
args.append("| less")
rawcmd = " ".join(args)
execcmd(rawcmd, show=True)
def _execsvnlook(self, cmd, *args, **kwargs):
execcmd_args = ["svnlook", cmd, self.repospath]
self._add_txnrev(execcmd_args, kwargs)
execcmd_args += args
execcmd_kwargs = {}
keywords = ["show", "noerror"]
for key in keywords:
if kwargs.has_key(key):
execcmd_kwargs[key] = kwargs[key]
return execcmd(*execcmd_args, **execcmd_kwargs)
def patch_spec(pkgdirurl, patchfile, log=""):
#FIXME use get_spec
svn = SVN()
tmpdir = tempfile.mktemp()
try:
geturl = layout.checkout_url(pkgdirurl, append_path="SPECS")
svn.checkout(geturl, tmpdir)
speclist = glob.glob(os.path.join(tmpdir, "*.spec"))
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
status, output = execcmd("patch", spec, patchfile)
if status != 0:
raise Error, "can't apply patch:\n%s\n" % output
else:
svn.commit(tmpdir, log="")
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def submit(urls, target, define=[], submithost=None, atonce=False, sid=None):
if submithost is None:
submithost = config.get("submit", "host")
if submithost is None:
# extract the submit host from the svn host
type, rest = urllib.splittype(pkgdirurl)
host, path = urllib.splithost(rest)
user, host = urllib.splituser(host)
submithost, port = urllib.splitport(host)
del type, user, port, path, rest
# runs a create-srpm in the server through ssh, which will make a
# copy of the rpm in the export directory
createsrpm = get_helper("create-srpm")
baseargs = ["ssh", submithost, createsrpm, "-t", target]
if not sid:
sid = uuid.uuid4()
define.append("sid=%s" % sid)
for entry in reversed(define):
baseargs.append("--define")
baseargs.append(entry)
cmdsargs = []
if len(urls) == 1:
# be compatible with server-side mgarepo versions older than 1.6.90
url, rev = layout.split_url_revision(urls[0])
baseargs.append("-r")
baseargs.append(str(rev))
baseargs.append(url)
cmdsargs.append(baseargs)
elif atonce:
cmdsargs.append(baseargs + urls)
else:
cmdsargs.extend((baseargs + [url]) for url in urls)
for cmdargs in cmdsargs:
command = subprocess.list2cmdline(cmdargs)
status, output = execcmd(command)
if status == 0:
print "Package submitted!"
else:
sys.stderr.write(output)
sys.exit(status)
def _execsvn(self, *args, **kwargs):
localcmds = ("add", "revert", "cleanup")
if not kwargs.get("show") and args[0] not in localcmds:
args = list(args)
args.append("--non-interactive")
else:
kwargs["geterr"] = True
kwargs["cleanerr"] = True
if kwargs.get("xml"):
args.append("--xml")
self._set_env()
svn_command = config.get("global", "svn-command", "svn")
cmdstr = svn_command + " " + " ".join(args)
try:
return execcmd(cmdstr, **kwargs)
except Error, e:
msg = None
if e.args:
if "Permission denied" in e.args[0]:
msg = ("It seems ssh-agent or ForwardAgent are not setup "
"or your username is wrong. See "
"https://wiki.mageia.org/en/Packagers_ssh"
" for more information.")
elif "authorization failed" in e.args[0]:
msg = ("Note that mgarepo does not support any HTTP "
"authenticated access.")
if kwargs.get("show") and \
not config.getbool("global", "verbose", 0):
# svn has already dumped error messages, we don't need to
# do it too
if msg:
sys.stderr.write("\n")
sys.stderr.write(msg)
sys.stderr.write("\n")
raise SilentError
elif msg:
raise Error, "%s\n%s" % (e, msg)
raise
def _getinfo(self):
cmdstr = "rpm -qp --nosignature --qf '%%{name} %%{epoch} %%{release} %%{version}' %s"
status, output = execcmd(cmdstr % self.filename)
self.name, self.epoch, self.release, self.version = output.split()
if self.epoch == "(none)":
self.epoch = None
def unpack(self, topdir):
execcmd(("rpm -i --nodeps --define '_sourcedir %s/SOURCES' " +
"--define '_specdir %s/SPECS' --define '_patchdir %s/SOURCES' %s")
% (topdir, topdir, topdir, self.filename))
"http://binrepo.mageia.org/")
url = mirror.normalize_path(url + "/" + sha1)
dest = os.path.join(topdir, 'SOURCES', filename)
if os.path.exists(dest):
if file_hash(dest) == sha1:
return 1
else:
raise Error, "File with incorrect sha1sum: %s" % dest
context = {"dest": dest, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError, e:
raise Error, "invalid variable %r in download-command "\
"configuration option" % e
try:
status, output = execcmd(cmd, show=True)
except Error, e:
os.unlink(dest)
raise Error, "Could not download file %s\n" % url
def download_binaries(topdir):
spath = sources_path(topdir)
if not os.path.exists(spath):
raise Error, "'%s' was not found" % spath
entries = parse_sources(spath)
for name, sha1 in entries.iteritems():
download_binary(topdir, sha1, name)
def binary_exists(sha1sum):
dlurl = config.get("binrepo", "download_url",
"http://binrepo.mageia.org/")
def get_srpm(pkgdirurl,
mode = "current",
targetdirs = None,
version = None,
release = None,
revision = None,
packager = "",
revname = 0,
svnlog = 0,
scripts = [],
submit = False,
template = None,
macros = [],
verbose = 0,
strict = False):
svn = SVN()
tmpdir = tempfile.mktemp()
topdir = "--define '_topdir %s'" % tmpdir
builddir = "--define '_builddir %s/%s'" % (tmpdir, "BUILD")
rpmdir = "--define '_rpmdir %s/%s'" % (tmpdir, "RPMS")
sourcedir = "--define '_sourcedir %s/%s'" % (tmpdir, "SOURCES")
specdir = "--define '_specdir %s/%s'" % (tmpdir, "SPECS")
srcrpmdir = "--define '_srcrpmdir %s/%s'" % (tmpdir, "SRPMS")
patchdir = "--define '_patchdir %s/%s'" % (tmpdir, "SOURCES")
try:
if mode == "version":
geturl = layout.checkout_url(pkgdirurl, version=version,
release=release)
elif mode == "pristine":
geturl = layout.checkout_url(pkgdirurl, pristine=True)
elif mode == "current" or mode == "revision":
#FIXME we should handle revisions specified using @REV
geturl = layout.checkout_url(pkgdirurl)
else:
raise Error, "unsupported get_srpm mode: %s" % mode
strict = strict or config.getbool("submit", "strict-revision", False)
if strict and not rev_touched_url(geturl, revision):
#FIXME would be nice to have the revision number even when
# revision is None
raise Error, "the revision %s does not change anything "\
"inside %s" % (revision or "HEAD", geturl)
mirror.info(geturl)
svn.export(geturl, tmpdir, rev=revision)
binrepo.download_binaries(tmpdir)
srpmsdir = os.path.join(tmpdir, "SRPMS")
os.mkdir(srpmsdir)
specsdir = os.path.join(tmpdir, "SPECS")
speclist = glob.glob(os.path.join(specsdir, "*.spec"))
if config.getbool("srpm", "run-prep", False):
makefile = os.path.join(tmpdir, "Makefile")
if os.path.exists(makefile):
execcmd("make", "-C", tmpdir, "srpm-prep")
if not speclist:
raise Error, "no spec files found"
spec = speclist[0]
if svnlog:
submit = not not revision
log.specfile_svn2rpm(pkgdirurl, spec, revision, submit=submit,
template=template, macros=macros, exported=tmpdir)
for script in scripts:
#FIXME revision can be "None"
status, output = execcmd(script, tmpdir, spec, str(revision),
noerror=1)
if status != 0:
raise Error, "script %s failed" % script
if packager:
packager = " --define 'packager %s'" % packager
defs = rpm_macros_defs(macros)
sourcecmd = config.get("helper", "rpmbuild", "rpmbuild")
execcmd("%s -bs --nodeps %s %s %s %s %s %s %s %s %s %s" %
(sourcecmd, topdir, builddir, rpmdir, sourcedir, specdir,
srcrpmdir, patchdir, packager, spec, defs))
# copy the generated SRPMs to their target locations
targetsrpms = []
urlrev = None
if revname:
urlrev = revision or layout.get_url_revision(geturl)
if not targetdirs:
targetdirs = (".",)
srpms = glob.glob(os.path.join(srpmsdir, "*.src.rpm"))
if not srpms:
# something fishy happened
raise Error, "no SRPMS were found at %s" % srpmsdir
for srpm in srpms:
name = os.path.basename(srpm)
if revname:
name = "@%s:%s" % (urlrev, name)
for targetdir in targetdirs:
newpath = os.path.join(targetdir, name)
targetsrpms.append(newpath)
if os.path.exists(newpath):
# should we warn?
os.unlink(newpath)
shutil.copy(srpm, newpath)
if verbose:
sys.stderr.write("Wrote: %s\n" % newpath)
return targetsrpms
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
def put_srpm(srpmfile, markrelease=False, striplog=True, branch=None,
baseurl=None, baseold=None, logmsg=None, rename=True):
svn = SVN()
srpm = SRPM(srpmfile)
tmpdir = tempfile.mktemp()
if baseurl:
pkgurl = mirror._joinurl(baseurl, srpm.name)
else:
pkgurl = layout.package_url(srpm.name, distro=branch,
mirrored=False)
print "Importing package to %s" % pkgurl
try:
if srpm.epoch:
version = "%s:%s" % (srpm.epoch, srpm.version)
else:
version = srpm.version
versionurl = "/".join([pkgurl, "releases", version])
releaseurl = "/".join([versionurl, srpm.release])
currenturl = "/".join([pkgurl, "current"])
currentdir = os.path.join(tmpdir, "current")
ret = svn.mkdir(pkgurl, noerror=1, log="Created package directory")
if ret or not svn.ls(currenturl, noerror=1):
svn.checkout(pkgurl, tmpdir)
svn.mkdir(os.path.join(tmpdir, "releases"))
svn.mkdir(currentdir)
svn.mkdir(os.path.join(currentdir, "SPECS"))
svn.mkdir(os.path.join(currentdir, "SOURCES"))
#svn.commit(tmpdir,log="Created package structure.")
version_exists = 1
else:
raise Error, "package already exists or error creating package directory"
specsdir = os.path.join(currentdir, "SPECS")
sourcesdir = os.path.join(currentdir, "SOURCES")
unpackdir = tempfile.mktemp()
os.mkdir(unpackdir)
try:
srpm.unpack(unpackdir)
uspecsdir = os.path.join(unpackdir, "SPECS")
usourcesdir = os.path.join(unpackdir, "SOURCES")
uspecsentries = os.listdir(uspecsdir)
usourcesentries = os.listdir(usourcesdir)
specsentries = os.listdir(specsdir)
sourcesentries = os.listdir(sourcesdir)
# Remove old entries
for entry in [x for x in specsentries
if x not in uspecsentries]:
if entry == ".svn":
continue
entrypath = os.path.join(specsdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
for entry in [x for x in sourcesentries
if x not in usourcesentries]:
if entry == ".svn":
continue
entrypath = os.path.join(sourcesdir, entry)
os.unlink(entrypath)
svn.remove(entrypath)
# Copy all files
execcmd("cp -rf", uspecsdir, currentdir)
execcmd("cp -rf", usourcesdir, currentdir)
# Add new entries
for entry in [x for x in uspecsentries
if x not in specsentries]:
entrypath = os.path.join(specsdir, entry)
svn.add(entrypath)
for entry in [x for x in usourcesentries
if x not in sourcesentries]:
entrypath = os.path.join(sourcesdir, entry)
if binrepo.is_binary(entrypath):
continue
svn.add(entrypath)
finally:
if os.path.isdir(unpackdir):
shutil.rmtree(unpackdir)
specs = glob.glob(os.path.join(specsdir, "*.spec"))
if not specs:
raise Error, "no spec file found on %s" % specsdir
if len(specs) > 1:
raise Error, "more than one spec file found on %s" % specsdir
specpath = specs[0]
if rename:
specfile = os.path.basename(specpath)
specname = specfile[:-len(".spec")]
if specname != srpm.name:
newname = srpm.name + ".spec"
newpath = os.path.join(specsdir, newname)
sys.stderr.write("warning: renaming spec file to '%s' "
"(use -n to disable it)\n" % (newname))
os.rename(specpath, newpath)
try:
svn.remove(specpath)
except Error:
# file not tracked
svn.revert(specpath)
svn.add(newpath)
specpath = newpath
if striplog:
specpath = specpath
fspec = open(specpath)
spec, chlog = log.split_spec_changelog(fspec)
fspec.close()
fspec = open(specpath, "w")
fspec.writelines(spec)
fspec.close()
chlog.seek(0, os.SEEK_END)
if chlog.tell() != 0:
chlog.seek(0)
#FIXME move it to layout.py
oldurl = baseold or config.get("log", "oldurl")
if oldurl == '.' or oldurl.startswith('./'):
pkgoldurl = os.path.join(pkgurl, oldurl)
else:
pkgoldurl = mirror._joinurl(oldurl, srpm.name)
svn.mkdir(pkgoldurl, noerror=1,
log="created old log directory for %s" % srpm.name)
logtmp = tempfile.mktemp()
try:
svn.checkout(pkgoldurl, logtmp)
miscpath = os.path.join(logtmp, "log")
fmisc = open(miscpath, "w+")
fmisc.writelines(chlog)
fmisc.close()
svn.add(miscpath)
svn.commit(logtmp,
log="imported old log for %s" % srpm.name)
finally:
if os.path.isdir(logtmp):
shutil.rmtree(logtmp)
binrepo.import_binaries(currentdir, srpm.name)
svn.commit(tmpdir,
log=logmsg or ("imported package %s" % srpm.name))
finally:
if os.path.isdir(tmpdir):
shutil.rmtree(tmpdir)
# Do revision and pristine tag copies
pristineurl = layout.checkout_url(pkgurl, pristine=True)
svn.remove(pristineurl, noerror=1,
log="Removing previous pristine/ directory.")
currenturl = layout.checkout_url(pkgurl)
svn.copy(currenturl, pristineurl,
log="Copying release %s-%s to pristine/ directory." %
(version, srpm.release))
if markrelease:
svn.copy(currenturl, releaseurl,
log="Copying release %s-%s to releases/ directory." %
(version, srpm.release))
if not (source in binrepoentries) and sourcesst.get(source)[1] == '?':
toadd.append(sourcepath)
elif not download and not os.path.isfile(sourcepath):
sys.stderr.write("warning: %s not found\n" % sourcepath)
elif download and not os.path.isfile(sourcepath):
print "%s not found, downloading from %s" % (sourcepath, url)
fmt = config.get("global", "download-command",
"wget -c -O '$dest' $url")
context = {"dest": sourcepath, "url": url}
try:
cmd = string.Template(fmt).substitute(context)
except KeyError, e:
raise Error, "invalid variable %r in download-command "\
"configuration option" % e
try:
status, output = execcmd(cmd, show=True)
except Error, e:
os.unlink(sourcepath)
raise Error, "Could not download file %s\n" % url
if os.path.isfile(sourcepath):
toadd.append(sourcepath)
else:
raise Error, "file not found: %s" % sourcepath
# rm entries not found in sources and still in svn
found = os.listdir(sourcesdir)
for entry in found:
if entry == ".svn" or entry == binrepo.SOURCES_FILE:
continue
status = sourcesst.get(entry)
path = os.path.join(sourcesdir, entry)
