def download(self, loc, ud, d):
"""
Fetch url
"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
oscupdatecmd = self._buildosccommand(ud, d, "update")
logger.info("Update "+ loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", oscupdatecmd)
bb.fetch2.check_network_access(d, oscupdatecmd)
runfetchcmd(oscupdatecmd, d)
else:
oscfetchcmd = self._buildosccommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", oscfetchcmd)
bb.fetch2.check_network_access(d, oscfetchcmd)
runfetchcmd(oscfetchcmd, d)
os.chdir(os.path.join(ud.pkgdir + ud.path))
# tar them up to a defined filename
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath])
def go(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
svnupdatecmd = self._buildsvncommand(ud, d, "update")
logger.info("Update " + loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", svnupdatecmd)
runfetchcmd(svnupdatecmd, d)
else:
svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", svnfetchcmd)
runfetchcmd(svnfetchcmd, d)
os.chdir(ud.pkgdir)
# tar them up to a defined filename
try:
runfetchcmd("tar --exclude '.svn' -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def obtain(fn, data):
import sys, bb
fn = bb.data.expand(fn, data)
localfn = bb.data.expand(localpath(fn, data), data)
if localfn != fn:
dldir = bb.data.getVar('DL_DIR', data, 1)
if not dldir:
bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: DL_DIR not defined")
return localfn
bb.mkdirhier(dldir)
try:
bb.fetch.init([fn], data)
except bb.fetch.NoMethodError:
(type, value, traceback) = sys.exc_info()
bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: no method: %s" % value)
return localfn
try:
bb.fetch.go(data)
except bb.fetch.MissingParameterError:
(type, value, traceback) = sys.exc_info()
bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: missing parameters: %s" % value)
return localfn
except bb.fetch.FetchError:
(type, value, traceback) = sys.exc_info()
bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: failed: %s" % value)
return localfn
return localfn
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
logger.debug(1, "BZR Update %s", loc)
os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
runfetchcmd(bzrcmd, d)
else:
bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
bzrcmd = self._buildbzrcommand(ud, d, "fetch")
logger.debug(1, "BZR Checkout %s", loc)
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", bzrcmd)
runfetchcmd(bzrcmd, d)
os.chdir(ud.pkgdir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
# tar them up to a defined filename
try:
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
logger.debug(1, "BZR Update %s", loc)
bb.fetch2.check_network_access(d, bzrcmd)
os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
runfetchcmd(bzrcmd, d)
else:
bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
bzrcmd = self._buildbzrcommand(ud, d, "fetch")
bb.fetch2.check_network_access(d, bzrcmd)
logger.debug(1, "BZR Checkout %s", loc)
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", bzrcmd)
runfetchcmd(bzrcmd, d)
os.chdir(ud.pkgdir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
# tar them up to a defined filename
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath])
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc)
os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path)))
runfetchcmd(bzrcmd, d)
else:
os.system("rm -rf %s" % os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)))
bzrcmd = self._buildbzrcommand(ud, d, "fetch")
bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Checkout %s" % loc)
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % bzrcmd)
runfetchcmd(bzrcmd, d)
os.chdir(ud.pkgdir)
# tar them up to a defined filename
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.pkgdir)), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def download(self, loc, ud, d):
"""Fetch urls"""
svkroot = ud.host + ud.path
svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)
if ud.revision:
svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module)
# create temp directory
localdata = data.createCopy(d)
data.update_data(localdata)
logger.debug(2, "Fetch: creating temporary directory")
bb.mkdirhier(data.expand('${WORKDIR}', localdata))
data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, True) or "false")
tmpfile = tmppipe.readline().strip()
if not tmpfile:
logger.error()
raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc)
# check out sources there
os.chdir(tmpfile)
logger.info("Fetch " + loc)
logger.debug(1, "Running %s", svkcmd)
runfetchcmd(svkcmd, d, cleanup = [tmpfile])
os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
# tar them up to a defined filename
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d, cleanup = [ud.localpath])
# cleanup
bb.utils.prunedir(tmpfile)
def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path))
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url)
runfetchcmd("repo sync", d)
os.chdir(codir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.repo' --exclude '.git'"
# Create a cache
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)
def go(self, loc, ud, d):
"""Fetch url"""
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
svnupdatecmd = self._buildsvncommand(ud, d, "update")
bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
# update sources there
os.chdir(ud.moddir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd)
runfetchcmd(svnupdatecmd, d)
else:
svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd)
runfetchcmd(svnfetchcmd, d)
os.chdir(ud.pkgdir)
# tar them up to a defined filename
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("repo sync", d)
os.chdir(codir)
# Create a cache
runfetchcmd("tar --exclude=.repo --exclude=.git -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
def go(self, d, urls = []):
"""Fetch urls"""
if not urls:
urls = self.urls
for loc in urls:
(type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))
tag = gettag(parm)
proto = getprotocol(parm)
gitsrcname = '%s%s' % (host, path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (tag)
codir = os.path.join(repodir, coname)
cofile = self.localpath(loc, d)
# tag=="master" must always update
if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % cofile)
continue
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
rungitcmd("tar -xzf %s" % (repofile),d)
else:
rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d)
os.chdir(repodir)
rungitcmd("git pull %s://%s%s" % (proto, host, path),d)
rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d)
rungitcmd("git prune-packed", d)
rungitcmd("git pack-redundant --all | xargs -r rm", d)
# Remove all but the .git directory
rungitcmd("rm * -Rf", d)
# old method of downloading tags
#rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)
os.chdir(repodir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)
if os.path.exists(codir):
prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
rungitcmd("git read-tree %s" % (tag),d)
rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)
def obtain(fn, data = bb.data.init()):
import sys, bb
fn = bb.data.expand(fn, data)
localfn = bb.data.expand(localpath(fn, data), data)
if localfn != fn:
dldir = bb.data.getVar('DL_DIR', data, 1)
if not dldir:
debug(1, "obtain: DL_DIR not defined")
return localfn
bb.mkdirhier(dldir)
try:
bb.fetch.init([fn])
except bb.fetch.NoMethodError:
(type, value, traceback) = sys.exc_info()
debug(1, "obtain: no method: %s" % value)
return localfn
try:
bb.fetch.go(data)
except bb.fetch.MissingParameterError:
(type, value, traceback) = sys.exc_info()
debug(1, "obtain: missing parameters: %s" % value)
return localfn
except bb.fetch.FetchError:
(type, value, traceback) = sys.exc_info()
debug(1, "obtain: failed: %s" % value)
return localfn
return localfn
def go(self, loc, ud, d):
"""
Fetch url
"""
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
oscupdatecmd = self._buildosccommand(ud, d, "update")
bb.msg.note(1, bb.msg.domain.Fetcher, "Update "+ loc)
# update sources there
os.chdir(ud.moddir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscupdatecmd)
runfetchcmd(oscupdatecmd, d)
else:
oscfetchcmd = self._buildosccommand(ud, d, "fetch")
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd)
runfetchcmd(oscfetchcmd, d)
os.chdir(os.path.join(ud.pkgdir + ud.path))
# tar them up to a defined filename
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
if not dirs:
dirs = (data.getVarFlag(func, 'dirs', d) or "").split()
for adir in dirs:
adir = data.expand(adir, d)
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
adir = data.expand(adir, d)
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.expand('${TOPDIR}', d)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
if data.getVarFlag(func, "python", d):
exec_func_python(func, d)
else:
exec_func_shell(func, d)
os.chdir(prevdir)
def download(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
svnupdatecmd = self._buildsvncommand(ud, d, "update")
logger.info("Update " + loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", svnupdatecmd)
bb.fetch2.check_network_access(d, svnupdatecmd)
runfetchcmd(svnupdatecmd, d)
else:
svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", svnfetchcmd)
bb.fetch2.check_network_access(d, svnfetchcmd)
runfetchcmd(svnfetchcmd, d)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.svn'"
os.chdir(ud.pkgdir)
# tar them up to a defined filename
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath])
def go(self, loc, ud, d):
"""
Fetch url
"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK):
oscupdatecmd = self._buildosccommand(ud, d, "update")
logger.info("Update "+ loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", oscupdatecmd)
runfetchcmd(oscupdatecmd, d)
else:
oscfetchcmd = self._buildosccommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", oscfetchcmd)
runfetchcmd(oscfetchcmd, d)
os.chdir(os.path.join(ud.pkgdir + ud.path))
# tar them up to a defined filename
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def mkstamp(task, d):
"""Creates/updates a stamp for a given task"""
stamp = data.getVar('STAMP', d)
if not stamp:
return
stamp = "%s.%s" % (data.expand(stamp, d), task)
mkdirhier(os.path.dirname(stamp))
open(stamp, "w+")
def go(self, loc, ud, d):
"""Fetch url"""
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
return
if ud.user:
username = ud.user + '@'
else:
username = ""
gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (ud.tag)
codir = os.path.join(repodir, coname)
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d)
os.chdir(repodir)
# Remove all but the .git directory
if not self._contains_ref(ud.tag, d):
runfetchcmd("rm * -Rf", d)
runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
runfetchcmd("git prune-packed", d)
runfetchcmd("git pack-redundant --all | xargs -r rm", d)
os.chdir(repodir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0":
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
if os.path.exists(codir):
bb.utils.prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
runfetchcmd("git read-tree %s" % (ud.tag), d)
runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
os.chdir(repodir)
bb.utils.prunedir(codir)
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
return
if ud.user:
username = ud.user + '@'
else:
username = ""
gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (ud.tag)
codir = os.path.join(repodir, coname)
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d)
os.chdir(repodir)
# Remove all but the .git directory
if not self._contains_ref(ud.tag, d):
runfetchcmd("rm * -Rf", d)
runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
runfetchcmd("git prune-packed", d)
runfetchcmd("git pack-redundant --all | xargs -r rm", d)
os.chdir(repodir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0":
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
if os.path.exists(codir):
bb.utils.prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
runfetchcmd("git read-tree %s" % (ud.tag), d)
runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
os.chdir(repodir)
bb.utils.prunedir(codir)
def go(self, loc, ud, d):
"""Fetch urls"""
if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(
d, ud.localfile):
return
svkroot = ud.host + ud.path
# pyflakes claims date is not known... it looks right
svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module)
if ud.revision:
svkcmd = "svk co -r %s/%s" % (ud.revision, svkroot, ud.module)
# create temp directory
localdata = data.createCopy(d)
data.update_data(localdata)
bb.msg.debug(2, bb.msg.domain.Fetcher,
"Fetch: creating temporary directory")
bb.mkdirhier(data.expand('${WORKDIR}', localdata))
data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX',
localdata), localdata)
tmppipe = os.popen(
data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
tmpfile = tmppipe.readline().strip()
if not tmpfile:
bb.msg.error(
bb.msg.domain.Fetcher,
"Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH."
)
raise FetchError(ud.module)
# check out sources there
os.chdir(tmpfile)
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd)
myret = os.system(svkcmd)
if myret != 0:
try:
os.rmdir(tmpfile)
except OSError:
pass
raise FetchError(ud.module)
os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
# tar them up to a defined filename
myret = os.system("tar -czf %s %s" %
(ud.localpath, os.path.basename(ud.module)))
if myret != 0:
try:
os.unlink(ud.localpath)
except OSError:
pass
raise FetchError(ud.module)
# cleanup
os.system('rm -rf %s' % tmpfile)
def exec_func(func, d, dirs=None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
flags = data.getVarFlags(func, d)
for item in [
'deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs',
'lockfiles', 'fakeroot'
]:
if not item in flags:
flags[item] = None
ispython = flags['python']
cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if dirs:
dirs = data.expand(dirs, d)
else:
dirs = (data.expand(flags['dirs'], d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.getVar('TOPDIR', d, True)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
locks = []
lockfiles = (data.expand(flags['lockfiles'], d) or "").split()
for lock in lockfiles:
locks.append(bb.utils.lockfile(lock))
if flags['python']:
exec_func_python(func, d)
else:
exec_func_shell(func, d, flags)
for lock in locks:
bb.utils.unlockfile(lock)
if os.path.exists(prevdir):
os.chdir(prevdir)
def go(self, loc, ud, d):
"""Fetch url"""
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
return
gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (ud.tag)
codir = os.path.join(repodir, coname)
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
runfetchcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir), d)
os.chdir(repodir)
# Remove all but the .git directory
runfetchcmd("rm * -Rf", d)
runfetchcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path), d)
runfetchcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path), d)
runfetchcmd("git prune-packed", d)
runfetchcmd("git pack-redundant --all | xargs -r rm", d)
# old method of downloading tags
#runfetchcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")), d)
os.chdir(repodir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
if os.path.exists(codir):
prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
runfetchcmd("git read-tree %s" % (ud.tag), d)
runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
os.chdir(repodir)
prunedir(codir)
def mkstamp(task, d):
"""Creates/updates a stamp for a given task"""
stamp = data.getVar('STAMP', d)
if not stamp:
return
stamp = "%s.%s" % (data.expand(stamp, d), task)
mkdirhier(os.path.dirname(stamp))
# Remove the file and recreate to force timestamp
# change on broken NFS filesystems
if os.access(stamp, os.F_OK):
os.remove(stamp)
f = open(stamp, "w")
f.close()
def __init__(self, d):
self.cachedir = bb.data.getVar("CACHE", d, True)
if self.cachedir in [None, '']:
bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'CACHE' variable.")
try:
os.stat(self.cachedir)
except OSError:
bb.mkdirhier(self.cachedir)
self.cachefile = os.path.join(self.cachedir,"bb_persist_data.sqlite3")
bb.msg.debug(1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile)
self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
def exec_func_shell(func, d, flags):
"""Execute a shell BB 'function' Returns true if execution was successful.
For this, it creates a bash shell script in the tmp dectory, writes the local
data into it and finally executes. The output of the shell will end in a log file and stdout.
Note on directory behavior. The 'dirs' varflag should contain a list
of the directories you need created prior to execution. The last
item in the list is where we will chdir/cd to.
"""
import sys
deps = flags['deps']
check = flags['check']
interact = flags['interactive']
if check in globals():
if globals()[check](func, deps):
return
global logfile
t = data.getVar('T', d, 1)
if not t:
return 0
mkdirhier(t)
logfile = "%s/log.%s.%s" % (t, func, str(os.getpid()))
runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
f = open(runfile, "w")
f.write("#!/bin/sh -e\n")
if bb.msg.debug_level['default'] > 0: f.write("set -x\n")
data.emit_env(f, d)
f.write("cd %s\n" % os.getcwd())
if func: f.write("%s\n" % func)
f.close()
os.chmod(runfile, 0775)
if not func:
bb.msg.error(bb.msg.domain.Build, "Function not specified")
raise FuncFailed()
# open logs
si = file('/dev/null', 'r')
try:
if bb.msg.debug_level['default'] > 0:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
except OSError, e:
bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
pass
def exec_func_shell(func, d):
"""Execute a shell BB 'function' Returns true if execution was successful.
For this, it creates a bash shell script in the tmp dectory, writes the local
data into it and finally executes. The output of the shell will end in a log file and stdout.
Note on directory behavior. The 'dirs' varflag should contain a list
of the directories you need created prior to execution. The last
item in the list is where we will chdir/cd to.
"""
import sys
deps = data.getVarFlag(func, 'deps', d)
check = data.getVarFlag(func, 'check', d)
interact = data.getVarFlag(func, 'interactive', d)
if check in globals():
if globals()[check](func, deps):
return
global logfile
t = data.getVar('T', d, 1)
if not t:
return 0
mkdirhier(t)
logfile = "%s/log.%s.%s" % (t, func, str(os.getpid()))
runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
f = open(runfile, "w")
f.write("#!/bin/sh -e\n")
if bb.msg.debug_level['default'] > 0: f.write("set -x\n")
data.emit_env(f, d)
f.write("cd %s\n" % os.getcwd())
if func: f.write("%s\n" % func)
f.close()
os.chmod(runfile, 0775)
if not func:
bb.msg.error(bb.msg.domain.Build, "Function not specified")
raise FuncFailed()
# open logs
si = file('/dev/null', 'r')
try:
if bb.msg.debug_level['default'] > 0:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
except OSError, e:
bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
pass
def go(self, loc, ud, d):
"""Fetch urls"""
svkroot = ud.host + ud.path
svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module)
if ud.revision:
svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module)
# create temp directory
localdata = data.createCopy(d)
data.update_data(localdata)
logger.debug(2, "Fetch: creating temporary directory")
bb.mkdirhier(data.expand('${WORKDIR}', localdata))
data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX',
localdata), localdata)
tmppipe = os.popen(
data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
tmpfile = tmppipe.readline().strip()
if not tmpfile:
logger.error(
"Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH."
)
raise FetchError(ud.module)
# check out sources there
os.chdir(tmpfile)
logger.info("Fetch " + loc)
logger.debug(1, "Running %s", svkcmd)
myret = os.system(svkcmd)
if myret != 0:
try:
os.rmdir(tmpfile)
except OSError:
pass
raise FetchError(ud.module)
os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
# tar them up to a defined filename
myret = os.system("tar -czf %s %s" %
(ud.localpath, os.path.basename(ud.module)))
if myret != 0:
try:
os.unlink(ud.localpath)
except OSError:
pass
raise FetchError(ud.module)
# cleanup
bb.utils.prunedir(tmpfile)
def go(self, loc, ud, d):
"""Fetch urls"""
if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
return
svkroot = ud.host + ud.path
# pyflakes claims date is not known... it looks right
svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module)
if ud.revision:
svkcmd = "svk co -r %s/%s" % (ud.revision, svkroot, ud.module)
# create temp directory
localdata = data.createCopy(d)
data.update_data(localdata)
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory")
bb.mkdirhier(data.expand('${WORKDIR}', localdata))
data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata)
tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
tmpfile = tmppipe.readline().strip()
if not tmpfile:
bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.")
raise FetchError(ud.module)
# check out sources there
os.chdir(tmpfile)
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd)
myret = os.system(svkcmd)
if myret != 0:
try:
os.rmdir(tmpfile)
except OSError:
pass
raise FetchError(ud.module)
os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module)))
# tar them up to a defined filename
myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)))
if myret != 0:
try:
os.unlink(ud.localpath)
except OSError:
pass
raise FetchError(ud.module)
# cleanup
os.system('rm -rf %s' % tmpfile)
def go(self, loc, ud, d):
"""Fetch url"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(
1, bb.msg.domain.Fetcher,
"%s already exists or was mirrored, skipping hg checkout." %
ud.localpath)
return
bb.msg.debug(
2, bb.msg.domain.Fetcher,
"Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
updatecmd = self._buildhgcommand(ud, d, "pull")
bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
# update sources there
os.chdir(ud.moddir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
runfetchcmd(updatecmd, d)
else:
fetchcmd = self._buildhgcommand(ud, d, "fetch")
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd)
runfetchcmd(fetchcmd, d)
# Even when we clone (fetch), we still need to update as hg's clone
# won't checkout the specified revision if its on a branch
updatecmd = self._buildhgcommand(ud, d, "update")
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
runfetchcmd(updatecmd, d)
os.chdir(ud.pkgdir)
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def go(self, loc, ud, d):
"""Fetch url"""
logger.debug(
2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
updatecmd = self._buildhgcommand(ud, d, "pull")
logger.info("Update " + loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", updatecmd)
runfetchcmd(updatecmd, d)
else:
fetchcmd = self._buildhgcommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", fetchcmd)
runfetchcmd(fetchcmd, d)
# Even when we clone (fetch), we still need to update as hg's clone
# won't checkout the specified revision if its on a branch
updatecmd = self._buildhgcommand(ud, d, "update")
os.chdir(ud.moddir)
logger.debug(1, "Running %s", updatecmd)
runfetchcmd(updatecmd, d)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.hg' --exclude '.hgrags'"
os.chdir(ud.pkgdir)
try:
runfetchcmd(
"tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def stamp_internal(task, d, file_name):
"""
Internal stamp helper function
Removes any stamp for the given task
Makes sure the stamp directory exists
Returns the stamp path+filename
"""
stamp = extract_stamp(d, file_name)
if not stamp:
return
stamp = "%s.%s" % (stamp, task)
mkdirhier(os.path.dirname(stamp))
# Remove the file and recreate to force timestamp
# change on broken NFS filesystems
if os.access(stamp, os.F_OK):
os.remove(stamp)
return stamp
def go(self, loc, ud, d):
"""
Fetch url
"""
# Try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(
1, bb.msg.domain.Fetcher,
"%s already exists or was mirrored, skipping osc checkout." %
ud.localpath)
return
bb.msg.debug(
2, bb.msg.domain.Fetcher,
"Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(
os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module),
os.R_OK):
oscupdatecmd = self._buildosccommand(ud, d, "update")
bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
# update sources there
os.chdir(ud.moddir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscupdatecmd)
runfetchcmd(oscupdatecmd, d)
else:
oscfetchcmd = self._buildosccommand(ud, d, "fetch")
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd)
runfetchcmd(oscfetchcmd, d)
os.chdir(os.path.join(ud.pkgdir + ud.path))
# tar them up to a defined filename
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def stamp_internal(task, d, file_name):
"""
Internal stamp helper function
Removes any stamp for the given task
Makes sure the stamp directory exists
Returns the stamp path+filename
"""
stamp = extract_stamp(d, file_name)
if not stamp:
return
stamp = "%s.%s" % (stamp, task)
mkdirhier(os.path.dirname(stamp))
# Remove the file and recreate to force timestamp
# change on broken NFS filesystems
if os.access(stamp, os.F_OK):
os.remove(stamp)
return stamp
def go(self, loc, ud, d):
"""Fetch url"""
logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
updatecmd = self._buildhgcommand(ud, d, "pull")
logger.info("Update " + loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", updatecmd)
runfetchcmd(updatecmd, d)
else:
fetchcmd = self._buildhgcommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", fetchcmd)
runfetchcmd(fetchcmd, d)
# Even when we clone (fetch), we still need to update as hg's clone
# won't checkout the specified revision if its on a branch
updatecmd = self._buildhgcommand(ud, d, "update")
os.chdir(ud.moddir)
logger.debug(1, "Running %s", updatecmd)
runfetchcmd(updatecmd, d)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.hg' --exclude '.hgrags'"
os.chdir(ud.pkgdir)
try:
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
cleandirs = (data.expand(data.getVarFlag(func, 'cleandirs', d), d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if not dirs:
dirs = (data.expand(data.getVarFlag(func, 'dirs', d), d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
adir = data.expand(adir, d)
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.expand('${TOPDIR}', d)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
locks = []
lockfiles = (data.expand(data.getVarFlag(func, 'lockfiles', d), d) or "").split()
for lock in lockfiles:
locks.append(bb.utils.lockfile(lock))
if data.getVarFlag(func, "python", d):
exec_func_python(func, d)
else:
exec_func_shell(func, d)
for lock in locks:
bb.utils.unlockfile(lock)
if os.path.exists(prevdir):
os.chdir(prevdir)
def go(self, loc, ud, d):
"""Fetch url"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath)
return
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK):
updatecmd = self._buildhgcommand(ud, d, "pull")
bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
# update sources there
os.chdir(ud.moddir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
runfetchcmd(updatecmd, d)
else:
fetchcmd = self._buildhgcommand(ud, d, "fetch")
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd)
runfetchcmd(fetchcmd, d)
# Even when we clone (fetch), we still need to update as hg's clone
# won't checkout the specified revision if its on a branch
updatecmd = self._buildhgcommand(ud, d, "update")
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd)
runfetchcmd(updatecmd, d)
os.chdir(ud.pkgdir)
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(
os.path.join(data.getVar("DL_DIR", d, True), ud.localfile),
os.R_OK):
logger.debug(
1,
"%s already exists (or was stashed). Skipping repo init / sync.",
ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(
data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
runfetchcmd(
"repo init -m %s -b %s -u %s://%s%s%s" %
(ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path),
d)
runfetchcmd("repo sync", d)
os.chdir(codir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.repo' --exclude '.git'"
# Create a cache
runfetchcmd(
"tar %s -czf %s %s" %
(tar_flags, ud.localpath, os.path.join(".", "*")), d)
def __init__(self, d):
self.cachedir = bb.data.getVar(
"PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True)
if self.cachedir in [None, '']:
bb.msg.fatal(
bb.msg.domain.PersistData,
"Please set the 'PERSISTENT_DIR' or 'CACHE' variable.")
try:
os.stat(self.cachedir)
except OSError:
bb.mkdirhier(self.cachedir)
self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3")
bb.msg.debug(
1, bb.msg.domain.PersistData,
"Using '%s' as the persistent data cache" % self.cachefile)
self.connection = sqlite3.connect(self.cachefile,
timeout=5,
isolation_level=None)
def do_finish():
bb.mkdirhier(build_dir)
os.chdir(pkgs_dir)
global seen
seen = []
for p in bb_get('PACKAGES').split():
pname = bb_get('NAME_%s' % p)
print "Package: %s (%s)" % (p, pname)
print "Description: ", bb_data['DESCRIPTION_'+p]
files = bb_data['FILES_'+p].split(" ")
pack_dir = pjoin(build_dir,p)
bb.mkdirhier(pack_dir)
install_files(files, pack_dir)
for data in DATAS:
if not bb_get(data+'_'+p):
if data == 'NAME':
bb_set(data+'_'+p, p)
else:
bb_set(data+'_'+p, bb_get(data+'_'+parent_pkg))
bb.mkdirhier(pjoin(pack_dir, 'CONTROL'))
write_control_file(pjoin(pack_dir, 'CONTROL'), p)
def do_finish():
bb.mkdirhier(build_dir)
os.chdir(pkgs_dir)
global seen
seen = []
for p in bb_get('PACKAGES').split():
pname = bb_get('NAME_%s' % p)
print "Package: %s (%s)" % (p, pname)
print "Description: ", bb_data['DESCRIPTION_' + p]
files = bb_data['FILES_' + p].split(" ")
pack_dir = pjoin(build_dir, p)
bb.mkdirhier(pack_dir)
install_files(files, pack_dir)
for data in DATAS:
if not bb_get(data + '_' + p):
if data == 'NAME':
bb_set(data + '_' + p, p)
else:
bb_set(data + '_' + p, bb_get(data + '_' + parent_pkg))
bb.mkdirhier(pjoin(pack_dir, 'CONTROL'))
write_control_file(pjoin(pack_dir, 'CONTROL'), p)
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(
os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'),
os.R_OK):
bzrcmd = self._buildbzrcommand(ud, d, "update")
logger.debug(1, "BZR Update %s", loc)
os.chdir(os.path.join(ud.pkgdir, os.path.basename(ud.path)))
runfetchcmd(bzrcmd, d)
else:
bb.utils.remove(
os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True)
bzrcmd = self._buildbzrcommand(ud, d, "fetch")
logger.debug(1, "BZR Checkout %s", loc)
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", bzrcmd)
runfetchcmd(bzrcmd, d)
os.chdir(ud.pkgdir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.bzr' --exclude '.bzrtags'"
# tar them up to a defined filename
try:
runfetchcmd(
"tar %s -czf %s %s" %
(tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def go(self, loc, ud, d):
"""Fetch url"""
logger.debug(
2, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
svnupdatecmd = self._buildsvncommand(ud, d, "update")
logger.info("Update " + loc)
# update sources there
os.chdir(ud.moddir)
logger.debug(1, "Running %s", svnupdatecmd)
runfetchcmd(svnupdatecmd, d)
else:
svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
logger.debug(1, "Running %s", svnfetchcmd)
runfetchcmd(svnfetchcmd, d)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.svn'"
os.chdir(ud.pkgdir)
# tar them up to a defined filename
try:
runfetchcmd(
"tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def download(self, loc, ud, d):
"""Fetch url"""
if ud.user:
username = ud.user + '@'
else:
username = ""
ud.repochanged = not os.path.exists(ud.fullmirror)
# If the checkout doesn't exist and the mirror tarball does, extract it
if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror):
bb.mkdirhier(ud.clonedir)
os.chdir(ud.clonedir)
runfetchcmd("tar -xzf %s" % (ud.fullmirror), d)
# If the repo still doesn't exist, fallback to cloning it
if not os.path.exists(ud.clonedir):
bb.fetch2.check_network_access(d, "git clone --bare %s%s" % (ud.host, ud.path))
runfetchcmd("%s clone --bare %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
os.chdir(ud.clonedir)
# Update the checkout if needed
needupdate = False
for name in ud.names:
if not self._contains_ref(ud.revisions[name], d):
needupdate = True
if needupdate:
bb.fetch2.check_network_access(d, "git fetch %s%s" % (ud.host, ud.path))
try:
runfetchcmd("%s remote prune origin" % ud.basecmd, d)
runfetchcmd("%s remote rm origin" % ud.basecmd, d)
except bb.fetch2.FetchError:
logger.debug(1, "No Origin")
runfetchcmd("%s remote add origin %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("%s fetch --all -t" % ud.basecmd, d)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
ud.repochanged = True
def go(self, loc, ud, d):
"""Fetch url"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(
1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath
)
return
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'")
if os.access(os.path.join(ud.moddir, ".svn"), os.R_OK):
svnupdatecmd = self._buildsvncommand(ud, d, "update")
bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
# update sources there
os.chdir(ud.moddir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd)
runfetchcmd(svnupdatecmd, d)
else:
svnfetchcmd = self._buildsvncommand(ud, d, "fetch")
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(ud.pkgdir)
os.chdir(ud.pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd)
runfetchcmd(svnfetchcmd, d)
os.chdir(ud.pkgdir)
# tar them up to a defined filename
try:
runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d)
except:
t, v, tb = sys.exc_info()
try:
os.unlink(ud.localpath)
except OSError:
pass
raise t, v, tb
def install_files(files, root):
import glob, errno, re, os
for file in files:
if os.path.isabs(file):
file = '.' + file
if not os.path.islink(file):
if os.path.isdir(file):
newfiles = [os.path.join(file, x) for x in os.listdir(file)]
if newfiles:
files += newfiles
continue
globbed = glob.glob(file)
if globbed:
if [file] != globbed:
if not file in globbed:
files += globbed
continue
else:
globbed.remove(file)
files += globbed
if (not os.path.islink(file)) and (not os.path.exists(file)):
continue
if file[-4:] == '.pyc':
continue
if file in seen:
continue
seen.append(file)
if os.path.isdir(file) and not os.path.islink(file):
bb.mkdirhier(os.path.join(root, file))
os.chmod(os.path.join(root, file), os.stat(file).st_mode)
continue
fpath = os.path.join(root, file)
dpath = os.path.dirname(fpath)
bb.mkdirhier(dpath)
ret = bb.copyfile(file, fpath)
if ret is False:
raise ("File population failed when copying %s to %s" %
(file, fpath))
def install_files(files, root):
import glob, errno, re,os
for file in files:
if os.path.isabs(file):
file = '.' + file
if not os.path.islink(file):
if os.path.isdir(file):
newfiles = [ os.path.join(file,x) for x in os.listdir(file) ]
if newfiles:
files += newfiles
continue
globbed = glob.glob(file)
if globbed:
if [ file ] != globbed:
if not file in globbed:
files += globbed
continue
else:
globbed.remove(file)
files += globbed
if (not os.path.islink(file)) and (not os.path.exists(file)):
continue
if file[-4:] == '.pyc':
continue
if file in seen:
continue
seen.append(file)
if os.path.isdir(file) and not os.path.islink(file):
bb.mkdirhier(os.path.join(root,file))
os.chmod(os.path.join(root,file), os.stat(file).st_mode)
continue
fpath = os.path.join(root,file)
dpath = os.path.dirname(fpath)
bb.mkdirhier(dpath)
ret = bb.copyfile(file, fpath)
if ret is False:
raise("File population failed when copying %s to %s" % (file, fpath))
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(
os.path.join(data.getVar("DL_DIR", d, True), ud.localfile),
os.R_OK):
bb.msg.debug(
1, bb.msg.domain.Fetcher,
"%s already exists (or was stashed). Skipping repo init / sync."
% ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(
data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
runfetchcmd(
"repo init -m %s -b %s -u %s://%s%s%s" %
(ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path),
d)
runfetchcmd("repo sync", d)
os.chdir(codir)
# Create a cache
runfetchcmd(
"tar --exclude=.repo --exclude=.git -czf %s %s" %
(ud.localpath, os.path.join(".", "*")), d)
def go(self, loc, ud, d):
# try to use the tarball stash
if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath)
return
method = "pserver"
if "method" in ud.parm:
method = ud.parm["method"]
localdir = ud.module
if "localdir" in ud.parm:
localdir = ud.parm["localdir"]
cvs_port = ""
if "port" in ud.parm:
cvs_port = ud.parm["port"]
cvs_rsh = None
if method == "ext":
if "rsh" in ud.parm:
cvs_rsh = ud.parm["rsh"]
if method == "dir":
cvsroot = ud.path
else:
cvsroot = ":" + method
cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True)
if cvsproxyhost:
cvsroot += ";proxy=" + cvsproxyhost
cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True)
if cvsproxyport:
cvsroot += ";proxyport=" + cvsproxyport
cvsroot += ":" + ud.user
if ud.pswd:
cvsroot += ":" + ud.pswd
cvsroot += "@" + ud.host + ":" + cvs_port + ud.path
options = []
if 'norecurse' in ud.parm:
options.append("-l")
if ud.date:
# treat YYYYMMDDHHMM specially for CVS
if len(ud.date) == 12:
options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12]))
else:
options.append("-D \"%s UTC\"" % ud.date)
if ud.tag:
options.append("-r %s" % ud.tag)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
data.setVar('CVSROOT', cvsroot, localdata)
data.setVar('CVSCOOPTS', " ".join(options), localdata)
data.setVar('CVSMODULE', ud.module, localdata)
cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
if cvs_rsh:
cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
# create module directory
bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory")
pkg = data.expand('${PN}', d)
pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
moddir = os.path.join(pkgdir,localdir)
if os.access(os.path.join(moddir,'CVS'), os.R_OK):
bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc)
# update sources there
os.chdir(moddir)
myret = os.system(cvsupdatecmd)
else:
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
# check out sources there
bb.mkdirhier(pkgdir)
os.chdir(pkgdir)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cvscmd)
myret = os.system(cvscmd)
if myret != 0 or not os.access(moddir, os.R_OK):
try:
os.rmdir(moddir)
except OSError:
pass
raise FetchError(ud.module)
# tar them up to a defined filename
if 'fullpath' in ud.parm:
os.chdir(pkgdir)
myret = os.system("tar -czf %s %s" % (ud.localpath, localdir))
else:
os.chdir(moddir)
os.chdir('..')
myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(moddir)))
if myret != 0:
try:
os.unlink(ud.localpath)
except OSError:
pass
raise FetchError(ud.module)
def go(self, loc, ud, d):
"""Fetch url"""
if ud.user:
username = ud.user + '@'
else:
username = ""
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
coname = '%s' % (ud.tag)
codir = os.path.join(ud.clonedir, coname)
if not os.path.exists(ud.clonedir):
try:
Fetch.try_mirrors(ud.mirrortarball)
bb.mkdirhier(ud.clonedir)
os.chdir(ud.clonedir)
runfetchcmd("tar -xzf %s" % (repofile), d)
except:
runfetchcmd(
"%s clone -n %s://%s%s%s %s" %
(ud.basecmd, ud.proto, username, ud.host, ud.path,
ud.clonedir), d)
os.chdir(ud.clonedir)
# Remove all but the .git directory
if not self._contains_ref(ud.tag, d):
runfetchcmd("rm * -Rf", d)
runfetchcmd(
"%s fetch %s://%s%s%s %s" %
(ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch),
d)
runfetchcmd(
"%s fetch --tags %s://%s%s%s" %
(ud.basecmd, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd,
d)
os.chdir(ud.clonedir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0" or 'fullclone' in ud.parm:
bb.msg.note(1, bb.msg.domain.Fetcher,
"Creating tarball of git repository")
runfetchcmd(
"tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*")),
d)
if 'fullclone' in ud.parm:
return
if os.path.exists(codir):
bb.utils.prunedir(codir)
subdir = ud.parm.get("subpath", "")
if subdir != "":
if subdir.endswith("/"):
subdirbase = os.path.basename(subdir[:-1])
else:
subdirbase = os.path.basename(subdir)
else:
subdirbase = ""
if subdir != "":
readpathspec = ":%s" % (subdir)
codir = os.path.join(codir, "git")
coprefix = os.path.join(codir, subdirbase, "")
else:
readpathspec = ""
coprefix = os.path.join(codir, "git", "")
bb.mkdirhier(codir)
os.chdir(ud.clonedir)
runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec),
d)
runfetchcmd(
"%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix),
d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher,
"Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*")),
d)
os.chdir(ud.clonedir)
bb.utils.prunedir(codir)
pkgs_dir = os.environ['packagingtmpdir']
print "searching for files in", pkgs_dir
#where this script will build packages
global build_dir
build_dir = os.environ['IPKGBUILDDIR']
print "temporary files goes in", build_dir
#######################################################################
#######################################################################
#Copied from bitbake (portage based) (c)
import bb
if not os.path.isdir(pkgs_dir):
bb.mkdirhier(pkgs_dir)
def install_files(files, root):
import glob, errno, re, os
for file in files:
if os.path.isabs(file):
file = '.' + file
if not os.path.islink(file):
if os.path.isdir(file):
newfiles = [os.path.join(file, x) for x in os.listdir(file)]
if newfiles:
files += newfiles
continue
globbed = glob.glob(file)
def go(self, loc, ud, d):
"""Fetch url"""
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(
1, bb.msg.domain.Fetcher,
"%s already exists (or was stashed). Skipping git checkout." %
ud.localpath)
return
if ud.user:
username = ud.user + '@'
else:
username = ""
gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (ud.tag)
if ud.subdir != "":
coname = '%s_%s' % (coname, ud.subdir.replace('/', '.'))
codir = os.path.join(repodir, coname)
""" A single repodir can be used for multiple checkouts. Protect against corruption. """
lf = bb.utils.lockfile("%s.%s" % (repofile, '.lock'))
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
runfetchcmd(
"git clone -n %s://%s%s%s %s" %
(ud.proto, username, ud.host, ud.path, repodir), d)
os.chdir(repodir)
# Remove all but the .git directory
if not self._contains_ref(ud.tag, d):
runfetchcmd("rm * -Rf", d)
runfetchcmd(
"git fetch %s://%s%s%s %s" %
(ud.proto, username, ud.host, ud.path, ud.branch), d)
runfetchcmd(
"git fetch --tags %s://%s%s%s" %
(ud.proto, username, ud.host, ud.path), d)
runfetchcmd("git prune-packed", d)
runfetchcmd("git pack-redundant --all | xargs -r rm", d)
os.chdir(repodir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0":
bb.msg.note(1, bb.msg.domain.Fetcher,
"Creating tarball of git repository")
runfetchcmd(
"tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*")),
d)
if os.path.exists(codir):
bb.utils.prunedir(codir)
if ud.subdir != "":
readpathspec = ":%s" % (ud.subdir)
subdir = os.path.basename(ud.subdir)
else:
readpathspec = ""
subdir = "git"
bb.mkdirhier(codir)
os.chdir(repodir)
runfetchcmd("git read-tree %s%s" % (ud.tag, readpathspec), d)
runfetchcmd(
"git checkout-index -q -f --prefix=%s -a" %
(os.path.join(codir, subdir, "")), d)
count = runfetchcmd("git rev-list %s -- | wc -l" % (ud.tag), d, True)
bb.utils.unlockfile(lf)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Checkins count: %s" % count)
f = open(os.path.join(subdir, '.git_revision_count'), 'w')
f.write(count)
f.close()
bb.msg.note(1, bb.msg.domain.Fetcher,
"Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*")),
d)
os.chdir(repodir)
bb.utils.prunedir(codir)
def go(self, loc, ud, d):
"""Fetch url"""
if ud.user:
username = ud.user + '@'
else:
username = ""
repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball)
coname = '%s' % (ud.tag)
codir = os.path.join(ud.clonedir, coname)
# If we have no existing clone and no mirror tarball, try and obtain one
if not os.path.exists(ud.clonedir) and not os.path.exists(repofile):
try:
Fetch.try_mirrors(ud.mirrortarball)
except:
pass
# If the checkout doesn't exist and the mirror tarball does, extract it
if not os.path.exists(ud.clonedir) and os.path.exists(repofile):
bb.mkdirhier(ud.clonedir)
os.chdir(ud.clonedir)
runfetchcmd("tar -xzf %s" % (repofile), d)
# If the repo still doesn't exist, fallback to cloning it
if not os.path.exists(ud.clonedir):
runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d)
os.chdir(ud.clonedir)
# Update the checkout if needed
if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm:
# Remove all but the .git directory
runfetchcmd("rm * -Rf", d)
if 'fullclone' in ud.parm:
runfetchcmd("%s fetch --all" % (ud.basecmd), d)
else:
runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d)
runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("%s prune-packed" % ud.basecmd, d)
runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d)
# Generate a mirror tarball if needed
os.chdir(ud.clonedir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0" or 'fullclone' in ud.parm:
logger.info("Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
if 'fullclone' in ud.parm:
return
if os.path.exists(codir):
bb.utils.prunedir(codir)
subdir = ud.parm.get("subpath", "")
if subdir != "":
if subdir.endswith("/"):
subdirbase = os.path.basename(subdir[:-1])
else:
subdirbase = os.path.basename(subdir)
else:
subdirbase = ""
if subdir != "":
readpathspec = ":%s" % (subdir)
codir = os.path.join(codir, "git")
coprefix = os.path.join(codir, subdirbase, "")
else:
readpathspec = ""
coprefix = os.path.join(codir, "git", "")
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d)
os.chdir(coprefix)
runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
else:
bb.mkdirhier(codir)
os.chdir(ud.clonedir)
runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d)
runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d)
os.chdir(codir)
logger.info("Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
os.chdir(ud.clonedir)
bb.utils.prunedir(codir)
def go(self, loc, ud, d):
method = ud.parm.get('method', 'pserver')
localdir = ud.parm.get('localdir', ud.module)
cvs_port = ud.parm.get('port', '')
cvs_rsh = None
if method == "ext":
if "rsh" in ud.parm:
cvs_rsh = ud.parm["rsh"]
if method == "dir":
cvsroot = ud.path
else:
cvsroot = ":" + method
cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True)
if cvsproxyhost:
cvsroot += ";proxy=" + cvsproxyhost
cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True)
if cvsproxyport:
cvsroot += ";proxyport=" + cvsproxyport
cvsroot += ":" + ud.user
if ud.pswd:
cvsroot += ":" + ud.pswd
cvsroot += "@" + ud.host + ":" + cvs_port + ud.path
options = []
if 'norecurse' in ud.parm:
options.append("-l")
if ud.date:
# treat YYYYMMDDHHMM specially for CVS
if len(ud.date) == 12:
options.append("-D \"%s %s:%s UTC\"" %
(ud.date[0:8], ud.date[8:10], ud.date[10:12]))
else:
options.append("-D \"%s UTC\"" % ud.date)
if ud.tag:
options.append("-r %s" % ud.tag)
localdata = data.createCopy(d)
data.setVar('OVERRIDES',
"cvs:%s" % data.getVar('OVERRIDES', localdata), localdata)
data.update_data(localdata)
data.setVar('CVSROOT', cvsroot, localdata)
data.setVar('CVSCOOPTS', " ".join(options), localdata)
data.setVar('CVSMODULE', ud.module, localdata)
cvscmd = data.getVar('FETCHCOMMAND', localdata, 1)
cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1)
if cvs_rsh:
cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd)
cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd)
# create module directory
logger.debug(2, "Fetch: checking for module directory")
pkg = data.expand('${PN}', d)
pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg)
moddir = os.path.join(pkgdir, localdir)
if os.access(os.path.join(moddir, 'CVS'), os.R_OK):
logger.info("Update " + loc)
# update sources there
os.chdir(moddir)
myret = os.system(cvsupdatecmd)
else:
logger.info("Fetch " + loc)
# check out sources there
bb.mkdirhier(pkgdir)
os.chdir(pkgdir)
logger.debug(1, "Running %s", cvscmd)
myret = os.system(cvscmd)
if myret != 0 or not os.access(moddir, os.R_OK):
try:
os.rmdir(moddir)
except OSError:
pass
raise FetchError(ud.module)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude 'CVS'"
# tar them up to a defined filename
if 'fullpath' in ud.parm:
os.chdir(pkgdir)
myret = os.system("tar %s -czf %s %s" %
(tar_flags, ud.localpath, localdir))
else:
os.chdir(moddir)
os.chdir('..')
myret = os.system(
"tar %s -czf %s %s" %
(tar_flags, ud.localpath, os.path.basename(moddir)))
if myret != 0:
try:
os.unlink(ud.localpath)
except OSError:
pass
raise FetchError(ud.module)
def exec_func(func, d, dirs=None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
flags = data.getVarFlags(func, d)
for item in [
'deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs',
'lockfiles', 'fakeroot'
]:
if not item in flags:
flags[item] = None
ispython = flags['python']
cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if dirs:
dirs = data.expand(dirs, d)
else:
dirs = (data.expand(flags['dirs'], d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
# Save current directory
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.getVar('TOPDIR', d, True)
# Setup logfiles
t = data.getVar('T', d, 1)
if not t:
bb.msg.fatal(bb.msg.domain.Build, "T not set")
mkdirhier(t)
logfile = "%s/log.%s.%s" % (t, func, str(os.getpid()))
runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
# Change to correct directory (if specified)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
# Handle logfiles
si = file('/dev/null', 'r')
try:
if bb.msg.debug_level['default'] > 0 or ispython:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
except OSError, e:
bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
pass
def go(self, loc, ud, d):
"""
Fetch urls
"""
# try to use the tarball stash
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(
1, bb.msg.domain.Fetcher,
"%s already exists or was mirrored, skipping perforce checkout."
% ud.localpath)
return
(host, depot, user, pswd, parm) = Perforce.doparse(loc, d)
if depot.find('/...') != -1:
path = depot[:depot.find('/...')]
else:
path = depot
if "module" in parm:
module = parm["module"]
else:
module = os.path.basename(path)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata),
localdata)
data.update_data(localdata)
# Get the p4 command
p4opt = ""
if user:
p4opt += " -u %s" % (user)
if pswd:
p4opt += " -P %s" % (pswd)
if host:
p4opt += " -p %s" % (host)
p4cmd = data.getVar('FETCHCOMMAND', localdata, 1)
# create temp directory
bb.msg.debug(2, bb.msg.domain.Fetcher,
"Fetch: creating temporary directory")
bb.mkdirhier(data.expand('${WORKDIR}', localdata))
data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX',
localdata), localdata)
tmppipe = os.popen(
data.getVar('MKTEMPDIRCMD', localdata, 1) or "false")
tmpfile = tmppipe.readline().strip()
if not tmpfile:
bb.error(
"Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH."
)
raise FetchError(module)
if "label" in parm:
depot = "%s@%s" % (depot, parm["label"])
else:
cset = Perforce.getcset(d, depot, host, user, pswd, parm)
depot = "%s@%s" % (depot, cset)
os.chdir(tmpfile)
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc)
bb.msg.note(1, bb.msg.domain.Fetcher,
"%s%s files %s" % (p4cmd, p4opt, depot))
p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot))
if not p4file:
bb.error("Fetch: unable to get the P4 files from %s" % (depot))
raise FetchError(module)
count = 0
for file in p4file:
list = file.split()
if list[2] == "delete":
continue
dest = list[0][len(path) + 1:]
where = dest.find("#")
os.system("%s%s print -o %s/%s %s" %
(p4cmd, p4opt, module, dest[:where], list[0]))
count = count + 1
if count == 0:
bb.error("Fetch: No files gathered from the P4 fetch")
raise FetchError(module)
myret = os.system("tar -czf %s %s" % (ud.localpath, module))
if myret != 0:
try:
os.unlink(ud.localpath)
except OSError:
pass
raise FetchError(module)
# cleanup
os.system('rm -rf %s' % tmpfile)
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
flags = data.getVarFlags(func, d)
for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']:
if not item in flags:
flags[item] = None
ispython = flags['python']
cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if dirs:
dirs = data.expand(dirs, d)
else:
dirs = (data.expand(flags['dirs'], d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
# Save current directory
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.getVar('TOPDIR', d, True)
# Setup logfiles
t = data.getVar('T', d, 1)
if not t:
bb.msg.fatal(bb.msg.domain.Build, "T not set")
mkdirhier(t)
# Gross hack, FIXME
import random
logfile = "%s/log.%s.%s.%s" % (t, func, str(os.getpid()),random.random())
runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
# Change to correct directory (if specified)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
# Handle logfiles
si = file('/dev/null', 'r')
try:
if bb.msg.debug_level['default'] > 0 or ispython:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
except OSError, e:
bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
pass
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
flags = data.getVarFlags(func, d)
for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']:
if not item in flags:
flags[item] = None
ispython = flags['python']
cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if dirs:
dirs = data.expand(dirs, d)
else:
dirs = (data.expand(flags['dirs'], d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.getVar('TOPDIR', d, True)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
locks = []
lockfiles = (data.expand(flags['lockfiles'], d) or "").split()
for lock in lockfiles:
locks.append(bb.utils.lockfile(lock))
if flags['python']:
exec_func_python(func, d)
else:
exec_func_shell(func, d, flags)
for lock in locks:
bb.utils.unlockfile(lock)
if os.path.exists(prevdir):
os.chdir(prevdir)
