def download(self, loc, ud, d): """ Fetch url """ logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): oscupdatecmd = self._buildosccommand(ud, d, "update") logger.info("Update "+ loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", oscupdatecmd) bb.fetch2.check_network_access(d, oscupdatecmd) runfetchcmd(oscupdatecmd, d) else: oscfetchcmd = self._buildosccommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", oscfetchcmd) bb.fetch2.check_network_access(d, oscfetchcmd) runfetchcmd(oscfetchcmd, d) os.chdir(os.path.join(ud.pkgdir + ud.path)) # tar them up to a defined filename runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d, cleanup = [ud.localpath])
def go(self, loc, ud, d): """Fetch url""" logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): svnupdatecmd = self._buildsvncommand(ud, d, "update") logger.info("Update " + loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", svnupdatecmd) runfetchcmd(svnupdatecmd, d) else: svnfetchcmd = self._buildsvncommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", svnfetchcmd) runfetchcmd(svnfetchcmd, d) os.chdir(ud.pkgdir) # tar them up to a defined filename try: runfetchcmd("tar --exclude '.svn' -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def obtain(fn, data): import sys, bb fn = bb.data.expand(fn, data) localfn = bb.data.expand(localpath(fn, data), data) if localfn != fn: dldir = bb.data.getVar('DL_DIR', data, 1) if not dldir: bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: DL_DIR not defined") return localfn bb.mkdirhier(dldir) try: bb.fetch.init([fn], data) except bb.fetch.NoMethodError: (type, value, traceback) = sys.exc_info() bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: no method: %s" % value) return localfn try: bb.fetch.go(data) except bb.fetch.MissingParameterError: (type, value, traceback) = sys.exc_info() bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: missing parameters: %s" % value) return localfn except bb.fetch.FetchError: (type, value, traceback) = sys.exc_info() bb.msg.debug(1, bb.msg.domain.Parsing, "obtain: failed: %s" % value) return localfn return localfn
def go(self, loc, ud, d): """Fetch url""" if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): bzrcmd = self._buildbzrcommand(ud, d, "update") logger.debug(1, "BZR Update %s", loc) os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path))) runfetchcmd(bzrcmd, d) else: bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True) bzrcmd = self._buildbzrcommand(ud, d, "fetch") logger.debug(1, "BZR Checkout %s", loc) bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", bzrcmd) runfetchcmd(bzrcmd, d) os.chdir(ud.pkgdir) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" # tar them up to a defined filename try: runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def download(self, loc, ud, d): """Fetch url""" if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): bzrcmd = self._buildbzrcommand(ud, d, "update") logger.debug(1, "BZR Update %s", loc) bb.fetch2.check_network_access(d, bzrcmd) os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path))) runfetchcmd(bzrcmd, d) else: bb.utils.remove(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True) bzrcmd = self._buildbzrcommand(ud, d, "fetch") bb.fetch2.check_network_access(d, bzrcmd) logger.debug(1, "BZR Checkout %s", loc) bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", bzrcmd) runfetchcmd(bzrcmd, d) os.chdir(ud.pkgdir) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" # tar them up to a defined filename runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d, cleanup = [ud.localpath])
def go(self, loc, ud, d): """Fetch url""" if os.access(os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): bzrcmd = self._buildbzrcommand(ud, d, "update") bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Update %s" % loc) os.chdir(os.path.join (ud.pkgdir, os.path.basename(ud.path))) runfetchcmd(bzrcmd, d) else: os.system("rm -rf %s" % os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir))) bzrcmd = self._buildbzrcommand(ud, d, "fetch") bb.msg.debug(1, bb.msg.domain.Fetcher, "BZR Checkout %s" % loc) bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % bzrcmd) runfetchcmd(bzrcmd, d) os.chdir(ud.pkgdir) # tar them up to a defined filename try: runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.pkgdir)), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def download(self, loc, ud, d): """Fetch urls""" svkroot = ud.host + ud.path svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module) if ud.revision: svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module) # create temp directory localdata = data.createCopy(d) data.update_data(localdata) logger.debug(2, "Fetch: creating temporary directory") bb.mkdirhier(data.expand('${WORKDIR}', localdata)) data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata) tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, True) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: logger.error() raise FetchError("Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.", loc) # check out sources there os.chdir(tmpfile) logger.info("Fetch " + loc) logger.debug(1, "Running %s", svkcmd) runfetchcmd(svkcmd, d, cleanup = [tmpfile]) os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) # tar them up to a defined filename runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module)), d, cleanup = [ud.localpath]) # cleanup bb.utils.prunedir(tmpfile)
def download(self, loc, ud, d): """Fetch url""" if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) return gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user: username = ud.user + "@" else: username = "" bb.mkdirhier(os.path.join(codir, "repo")) os.chdir(os.path.join(codir, "repo")) if not os.path.exists(os.path.join(codir, "repo", ".repo")): bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path)) runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) bb.fetch2.check_network_access(d, "repo sync %s" % ud.url) runfetchcmd("repo sync", d) os.chdir(codir) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.repo' --exclude '.git'" # Create a cache runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)
def go(self, loc, ud, d): """Fetch url""" bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): svnupdatecmd = self._buildsvncommand(ud, d, "update") bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(ud.moddir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd) runfetchcmd(svnupdatecmd, d) else: svnfetchcmd = self._buildsvncommand(ud, d, "fetch") bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd) runfetchcmd(svnfetchcmd, d) os.chdir(ud.pkgdir) # tar them up to a defined filename try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def go(self, loc, ud, d): """Fetch url""" if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) return gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo") codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user: username = ud.user + "@" else: username = "" bb.mkdirhier(os.path.join(codir, "repo")) os.chdir(os.path.join(codir, "repo")) if not os.path.exists(os.path.join(codir, "repo", ".repo")): runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) runfetchcmd("repo sync", d) os.chdir(codir) # Create a cache runfetchcmd("tar --exclude=.repo --exclude=.git -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
def go(self, d, urls = []): """Fetch urls""" if not urls: urls = self.urls for loc in urls: (type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d)) tag = gettag(parm) proto = getprotocol(parm) gitsrcname = '%s%s' % (host, path.replace('/', '.')) repofilename = 'git_%s.tar.gz' % (gitsrcname) repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) coname = '%s' % (tag) codir = os.path.join(repodir, coname) cofile = self.localpath(loc, d) # tag=="master" must always update if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)): bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % cofile) continue if not os.path.exists(repodir): if Fetch.try_mirror(d, repofilename): bb.mkdirhier(repodir) os.chdir(repodir) rungitcmd("tar -xzf %s" % (repofile),d) else: rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d) os.chdir(repodir) rungitcmd("git pull %s://%s%s" % (proto, host, path),d) rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d) rungitcmd("git prune-packed", d) rungitcmd("git pack-redundant --all | xargs -r rm", d) # Remove all but the .git directory rungitcmd("rm * -Rf", d) # old method of downloading tags #rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d) os.chdir(repodir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d) if os.path.exists(codir): prunedir(codir) bb.mkdirhier(codir) os.chdir(repodir) rungitcmd("git read-tree %s" % (tag),d) rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d) os.chdir(codir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)
def obtain(fn, data = bb.data.init()): import sys, bb fn = bb.data.expand(fn, data) localfn = bb.data.expand(localpath(fn, data), data) if localfn != fn: dldir = bb.data.getVar('DL_DIR', data, 1) if not dldir: debug(1, "obtain: DL_DIR not defined") return localfn bb.mkdirhier(dldir) try: bb.fetch.init([fn]) except bb.fetch.NoMethodError: (type, value, traceback) = sys.exc_info() debug(1, "obtain: no method: %s" % value) return localfn try: bb.fetch.go(data) except bb.fetch.MissingParameterError: (type, value, traceback) = sys.exc_info() debug(1, "obtain: missing parameters: %s" % value) return localfn except bb.fetch.FetchError: (type, value, traceback) = sys.exc_info() debug(1, "obtain: failed: %s" % value) return localfn return localfn
def go(self, loc, ud, d): """ Fetch url """ bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): oscupdatecmd = self._buildosccommand(ud, d, "update") bb.msg.note(1, bb.msg.domain.Fetcher, "Update "+ loc) # update sources there os.chdir(ud.moddir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscupdatecmd) runfetchcmd(oscupdatecmd, d) else: oscfetchcmd = self._buildosccommand(ud, d, "fetch") bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd) runfetchcmd(oscfetchcmd, d) os.chdir(os.path.join(ud.pkgdir + ud.path)) # tar them up to a defined filename try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def exec_func(func, d, dirs = None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: return if not dirs: dirs = (data.getVarFlag(func, 'dirs', d) or "").split() for adir in dirs: adir = data.expand(adir, d) mkdirhier(adir) if len(dirs) > 0: adir = dirs[-1] else: adir = data.getVar('B', d, 1) adir = data.expand(adir, d) try: prevdir = os.getcwd() except OSError: prevdir = data.expand('${TOPDIR}', d) if adir and os.access(adir, os.F_OK): os.chdir(adir) if data.getVarFlag(func, "python", d): exec_func_python(func, d) else: exec_func_shell(func, d) os.chdir(prevdir)
def download(self, loc, ud, d): """Fetch url""" logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): svnupdatecmd = self._buildsvncommand(ud, d, "update") logger.info("Update " + loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", svnupdatecmd) bb.fetch2.check_network_access(d, svnupdatecmd) runfetchcmd(svnupdatecmd, d) else: svnfetchcmd = self._buildsvncommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", svnfetchcmd) bb.fetch2.check_network_access(d, svnfetchcmd) runfetchcmd(svnfetchcmd, d) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.svn'" os.chdir(ud.pkgdir) # tar them up to a defined filename runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d, cleanup = [ud.localpath])
def go(self, loc, ud, d): """ Fetch url """ logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): oscupdatecmd = self._buildosccommand(ud, d, "update") logger.info("Update "+ loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", oscupdatecmd) runfetchcmd(oscupdatecmd, d) else: oscfetchcmd = self._buildosccommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", oscfetchcmd) runfetchcmd(oscfetchcmd, d) os.chdir(os.path.join(ud.pkgdir + ud.path)) # tar them up to a defined filename try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def mkstamp(task, d): """Creates/updates a stamp for a given task""" stamp = data.getVar('STAMP', d) if not stamp: return stamp = "%s.%s" % (data.expand(stamp, d), task) mkdirhier(os.path.dirname(stamp)) open(stamp, "w+")
def go(self, loc, ud, d): """Fetch url""" if Fetch.try_mirror(d, ud.localfile): bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) return if ud.user: username = ud.user + '@' else: username = "" gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) repofilename = 'git_%s.tar.gz' % (gitsrcname) repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) coname = '%s' % (ud.tag) codir = os.path.join(repodir, coname) if not os.path.exists(repodir): if Fetch.try_mirror(d, repofilename): bb.mkdirhier(repodir) os.chdir(repodir) runfetchcmd("tar -xzf %s" % (repofile), d) else: runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d) os.chdir(repodir) # Remove all but the .git directory if not self._contains_ref(ud.tag, d): runfetchcmd("rm * -Rf", d) runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d) runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d) runfetchcmd("git prune-packed", d) runfetchcmd("git pack-redundant --all | xargs -r rm", d) os.chdir(repodir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) if mirror_tarballs != "0": bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) if os.path.exists(codir): bb.utils.prunedir(codir) bb.mkdirhier(codir) os.chdir(repodir) runfetchcmd("git read-tree %s" % (ud.tag), d) runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d) os.chdir(codir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) os.chdir(repodir) bb.utils.prunedir(codir)
def go(self, loc, ud, d): """Fetch url""" if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) return if ud.user: username = ud.user + '@' else: username = "" gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) repofilename = 'git_%s.tar.gz' % (gitsrcname) repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) coname = '%s' % (ud.tag) codir = os.path.join(repodir, coname) if not os.path.exists(repodir): if Fetch.try_mirror(d, repofilename): bb.mkdirhier(repodir) os.chdir(repodir) runfetchcmd("tar -xzf %s" % (repofile), d) else: runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d) os.chdir(repodir) # Remove all but the .git directory if not self._contains_ref(ud.tag, d): runfetchcmd("rm * -Rf", d) runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d) runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d) runfetchcmd("git prune-packed", d) runfetchcmd("git pack-redundant --all | xargs -r rm", d) os.chdir(repodir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) if mirror_tarballs != "0": bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) if os.path.exists(codir): bb.utils.prunedir(codir) bb.mkdirhier(codir) os.chdir(repodir) runfetchcmd("git read-tree %s" % (ud.tag), d) runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d) os.chdir(codir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) os.chdir(repodir) bb.utils.prunedir(codir)
def go(self, loc, ud, d): """Fetch urls""" if not self.forcefetch(loc, ud, d) and Fetch.try_mirror( d, ud.localfile): return svkroot = ud.host + ud.path # pyflakes claims date is not known... it looks right svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module) if ud.revision: svkcmd = "svk co -r %s/%s" % (ud.revision, svkroot, ud.module) # create temp directory localdata = data.createCopy(d) data.update_data(localdata) bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory") bb.mkdirhier(data.expand('${WORKDIR}', localdata)) data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata) tmppipe = os.popen( data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: bb.msg.error( bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH." ) raise FetchError(ud.module) # check out sources there os.chdir(tmpfile) bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd) myret = os.system(svkcmd) if myret != 0: try: os.rmdir(tmpfile) except OSError: pass raise FetchError(ud.module) os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) # tar them up to a defined filename myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module))) if myret != 0: try: os.unlink(ud.localpath) except OSError: pass raise FetchError(ud.module) # cleanup os.system('rm -rf %s' % tmpfile)
def exec_func(func, d, dirs=None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: return flags = data.getVarFlags(func, d) for item in [ 'deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot' ]: if not item in flags: flags[item] = None ispython = flags['python'] cleandirs = (data.expand(flags['cleandirs'], d) or "").split() for cdir in cleandirs: os.system("rm -rf %s" % cdir) if dirs: dirs = data.expand(dirs, d) else: dirs = (data.expand(flags['dirs'], d) or "").split() for adir in dirs: mkdirhier(adir) if len(dirs) > 0: adir = dirs[-1] else: adir = data.getVar('B', d, 1) try: prevdir = os.getcwd() except OSError: prevdir = data.getVar('TOPDIR', d, True) if adir and os.access(adir, os.F_OK): os.chdir(adir) locks = [] lockfiles = (data.expand(flags['lockfiles'], d) or "").split() for lock in lockfiles: locks.append(bb.utils.lockfile(lock)) if flags['python']: exec_func_python(func, d) else: exec_func_shell(func, d, flags) for lock in locks: bb.utils.unlockfile(lock) if os.path.exists(prevdir): os.chdir(prevdir)
def go(self, loc, ud, d): """Fetch url""" if Fetch.try_mirror(d, ud.localfile): bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) return gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) repofilename = 'git_%s.tar.gz' % (gitsrcname) repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) coname = '%s' % (ud.tag) codir = os.path.join(repodir, coname) if not os.path.exists(repodir): if Fetch.try_mirror(d, repofilename): bb.mkdirhier(repodir) os.chdir(repodir) runfetchcmd("tar -xzf %s" % (repofile), d) else: runfetchcmd("git clone -n %s://%s%s %s" % (ud.proto, ud.host, ud.path, repodir), d) os.chdir(repodir) # Remove all but the .git directory runfetchcmd("rm * -Rf", d) runfetchcmd("git pull %s://%s%s" % (ud.proto, ud.host, ud.path), d) runfetchcmd("git pull --tags %s://%s%s" % (ud.proto, ud.host, ud.path), d) runfetchcmd("git prune-packed", d) runfetchcmd("git pack-redundant --all | xargs -r rm", d) # old method of downloading tags #runfetchcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (ud.host, ud.path, os.path.join(repodir, ".git", "")), d) os.chdir(repodir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) if os.path.exists(codir): prunedir(codir) bb.mkdirhier(codir) os.chdir(repodir) runfetchcmd("git read-tree %s" % (ud.tag), d) runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d) os.chdir(codir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) os.chdir(repodir) prunedir(codir)
def mkstamp(task, d): """Creates/updates a stamp for a given task""" stamp = data.getVar('STAMP', d) if not stamp: return stamp = "%s.%s" % (data.expand(stamp, d), task) mkdirhier(os.path.dirname(stamp)) # Remove the file and recreate to force timestamp # change on broken NFS filesystems if os.access(stamp, os.F_OK): os.remove(stamp) f = open(stamp, "w") f.close()
def __init__(self, d): self.cachedir = bb.data.getVar("CACHE", d, True) if self.cachedir in [None, '']: bb.msg.fatal(bb.msg.domain.PersistData, "Please set the 'CACHE' variable.") try: os.stat(self.cachedir) except OSError: bb.mkdirhier(self.cachedir) self.cachefile = os.path.join(self.cachedir,"bb_persist_data.sqlite3") bb.msg.debug(1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile) self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
def exec_func_shell(func, d, flags): """Execute a shell BB 'function' Returns true if execution was successful. For this, it creates a bash shell script in the tmp dectory, writes the local data into it and finally executes. The output of the shell will end in a log file and stdout. Note on directory behavior. The 'dirs' varflag should contain a list of the directories you need created prior to execution. The last item in the list is where we will chdir/cd to. """ import sys deps = flags['deps'] check = flags['check'] interact = flags['interactive'] if check in globals(): if globals()[check](func, deps): return global logfile t = data.getVar('T', d, 1) if not t: return 0 mkdirhier(t) logfile = "%s/log.%s.%s" % (t, func, str(os.getpid())) runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) f = open(runfile, "w") f.write("#!/bin/sh -e\n") if bb.msg.debug_level['default'] > 0: f.write("set -x\n") data.emit_env(f, d) f.write("cd %s\n" % os.getcwd()) if func: f.write("%s\n" % func) f.close() os.chmod(runfile, 0775) if not func: bb.msg.error(bb.msg.domain.Build, "Function not specified") raise FuncFailed() # open logs si = file('/dev/null', 'r') try: if bb.msg.debug_level['default'] > 0: so = os.popen("tee \"%s\"" % logfile, "w") else: so = file(logfile, 'w') except OSError, e: bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) pass
def exec_func_shell(func, d): """Execute a shell BB 'function' Returns true if execution was successful. For this, it creates a bash shell script in the tmp dectory, writes the local data into it and finally executes. The output of the shell will end in a log file and stdout. Note on directory behavior. The 'dirs' varflag should contain a list of the directories you need created prior to execution. The last item in the list is where we will chdir/cd to. """ import sys deps = data.getVarFlag(func, 'deps', d) check = data.getVarFlag(func, 'check', d) interact = data.getVarFlag(func, 'interactive', d) if check in globals(): if globals()[check](func, deps): return global logfile t = data.getVar('T', d, 1) if not t: return 0 mkdirhier(t) logfile = "%s/log.%s.%s" % (t, func, str(os.getpid())) runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) f = open(runfile, "w") f.write("#!/bin/sh -e\n") if bb.msg.debug_level['default'] > 0: f.write("set -x\n") data.emit_env(f, d) f.write("cd %s\n" % os.getcwd()) if func: f.write("%s\n" % func) f.close() os.chmod(runfile, 0775) if not func: bb.msg.error(bb.msg.domain.Build, "Function not specified") raise FuncFailed() # open logs si = file('/dev/null', 'r') try: if bb.msg.debug_level['default'] > 0: so = os.popen("tee \"%s\"" % logfile, "w") else: so = file(logfile, 'w') except OSError, e: bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) pass
def go(self, loc, ud, d): """Fetch urls""" svkroot = ud.host + ud.path svkcmd = "svk co -r {%s} %s/%s" % (ud.date, svkroot, ud.module) if ud.revision: svkcmd = "svk co -r %s %s/%s" % (ud.revision, svkroot, ud.module) # create temp directory localdata = data.createCopy(d) data.update_data(localdata) logger.debug(2, "Fetch: creating temporary directory") bb.mkdirhier(data.expand('${WORKDIR}', localdata)) data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata) tmppipe = os.popen( data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: logger.error( "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH." ) raise FetchError(ud.module) # check out sources there os.chdir(tmpfile) logger.info("Fetch " + loc) logger.debug(1, "Running %s", svkcmd) myret = os.system(svkcmd) if myret != 0: try: os.rmdir(tmpfile) except OSError: pass raise FetchError(ud.module) os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) # tar them up to a defined filename myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module))) if myret != 0: try: os.unlink(ud.localpath) except OSError: pass raise FetchError(ud.module) # cleanup bb.utils.prunedir(tmpfile)
def go(self, loc, ud, d): """Fetch urls""" if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): return svkroot = ud.host + ud.path # pyflakes claims date is not known... it looks right svkcmd = "svk co -r {%s} %s/%s" % (date, svkroot, ud.module) if ud.revision: svkcmd = "svk co -r %s/%s" % (ud.revision, svkroot, ud.module) # create temp directory localdata = data.createCopy(d) data.update_data(localdata) bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory") bb.mkdirhier(data.expand('${WORKDIR}', localdata)) data.setVar('TMPBASE', data.expand('${WORKDIR}/oesvk.XXXXXX', localdata), localdata) tmppipe = os.popen(data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: bb.msg.error(bb.msg.domain.Fetcher, "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH.") raise FetchError(ud.module) # check out sources there os.chdir(tmpfile) bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svkcmd) myret = os.system(svkcmd) if myret != 0: try: os.rmdir(tmpfile) except OSError: pass raise FetchError(ud.module) os.chdir(os.path.join(tmpfile, os.path.dirname(ud.module))) # tar them up to a defined filename myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(ud.module))) if myret != 0: try: os.unlink(ud.localpath) except OSError: pass raise FetchError(ud.module) # cleanup os.system('rm -rf %s' % tmpfile)
def go(self, loc, ud, d): """Fetch url""" # try to use the tarball stash if Fetch.try_mirror(d, ud.localfile): bb.msg.debug( 1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath) return bb.msg.debug( 2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): updatecmd = self._buildhgcommand(ud, d, "pull") bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(ud.moddir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd) runfetchcmd(updatecmd, d) else: fetchcmd = self._buildhgcommand(ud, d, "fetch") bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd) runfetchcmd(fetchcmd, d) # Even when we clone (fetch), we still need to update as hg's clone # won't checkout the specified revision if its on a branch updatecmd = self._buildhgcommand(ud, d, "update") bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd) runfetchcmd(updatecmd, d) os.chdir(ud.pkgdir) try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def go(self, loc, ud, d): """Fetch url""" logger.debug( 2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): updatecmd = self._buildhgcommand(ud, d, "pull") logger.info("Update " + loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", updatecmd) runfetchcmd(updatecmd, d) else: fetchcmd = self._buildhgcommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", fetchcmd) runfetchcmd(fetchcmd, d) # Even when we clone (fetch), we still need to update as hg's clone # won't checkout the specified revision if its on a branch updatecmd = self._buildhgcommand(ud, d, "update") os.chdir(ud.moddir) logger.debug(1, "Running %s", updatecmd) runfetchcmd(updatecmd, d) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.hg' --exclude '.hgrags'" os.chdir(ud.pkgdir) try: runfetchcmd( "tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def stamp_internal(task, d, file_name): """ Internal stamp helper function Removes any stamp for the given task Makes sure the stamp directory exists Returns the stamp path+filename """ stamp = extract_stamp(d, file_name) if not stamp: return stamp = "%s.%s" % (stamp, task) mkdirhier(os.path.dirname(stamp)) # Remove the file and recreate to force timestamp # change on broken NFS filesystems if os.access(stamp, os.F_OK): os.remove(stamp) return stamp
def go(self, loc, ud, d): """ Fetch url """ # Try to use the tarball stash if Fetch.try_mirror(d, ud.localfile): bb.msg.debug( 1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping osc checkout." % ud.localpath) return bb.msg.debug( 2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access( os.path.join(data.expand('${OSCDIR}', d), ud.path, ud.module), os.R_OK): oscupdatecmd = self._buildosccommand(ud, d, "update") bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(ud.moddir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscupdatecmd) runfetchcmd(oscupdatecmd, d) else: oscfetchcmd = self._buildosccommand(ud, d, "fetch") bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % oscfetchcmd) runfetchcmd(oscfetchcmd, d) os.chdir(os.path.join(ud.pkgdir + ud.path)) # tar them up to a defined filename try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def go(self, loc, ud, d): """Fetch url""" logger.debug(2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): updatecmd = self._buildhgcommand(ud, d, "pull") logger.info("Update " + loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", updatecmd) runfetchcmd(updatecmd, d) else: fetchcmd = self._buildhgcommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", fetchcmd) runfetchcmd(fetchcmd, d) # Even when we clone (fetch), we still need to update as hg's clone # won't checkout the specified revision if its on a branch updatecmd = self._buildhgcommand(ud, d, "update") os.chdir(ud.moddir) logger.debug(1, "Running %s", updatecmd) runfetchcmd(updatecmd, d) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.hg' --exclude '.hgrags'" os.chdir(ud.pkgdir) try: runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def exec_func(func, d, dirs = None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: return cleandirs = (data.expand(data.getVarFlag(func, 'cleandirs', d), d) or "").split() for cdir in cleandirs: os.system("rm -rf %s" % cdir) if not dirs: dirs = (data.expand(data.getVarFlag(func, 'dirs', d), d) or "").split() for adir in dirs: mkdirhier(adir) if len(dirs) > 0: adir = dirs[-1] else: adir = data.getVar('B', d, 1) adir = data.expand(adir, d) try: prevdir = os.getcwd() except OSError: prevdir = data.expand('${TOPDIR}', d) if adir and os.access(adir, os.F_OK): os.chdir(adir) locks = [] lockfiles = (data.expand(data.getVarFlag(func, 'lockfiles', d), d) or "").split() for lock in lockfiles: locks.append(bb.utils.lockfile(lock)) if data.getVarFlag(func, "python", d): exec_func_python(func, d) else: exec_func_shell(func, d) for lock in locks: bb.utils.unlockfile(lock) if os.path.exists(prevdir): os.chdir(prevdir)
def go(self, loc, ud, d): """Fetch url""" # try to use the tarball stash if Fetch.try_mirror(d, ud.localfile): bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping hg checkout." % ud.localpath) return bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): updatecmd = self._buildhgcommand(ud, d, "pull") bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(ud.moddir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd) runfetchcmd(updatecmd, d) else: fetchcmd = self._buildhgcommand(ud, d, "fetch") bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % fetchcmd) runfetchcmd(fetchcmd, d) # Even when we clone (fetch), we still need to update as hg's clone # won't checkout the specified revision if its on a branch updatecmd = self._buildhgcommand(ud, d, "update") bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % updatecmd) runfetchcmd(updatecmd, d) os.chdir(ud.pkgdir) try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def go(self, loc, ud, d): """Fetch url""" if os.access( os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): logger.debug( 1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath) return gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join( data.getVar("DL_DIR", d, True), "repo") codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user: username = ud.user + "@" else: username = "" bb.mkdirhier(os.path.join(codir, "repo")) os.chdir(os.path.join(codir, "repo")) if not os.path.exists(os.path.join(codir, "repo", ".repo")): runfetchcmd( "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) runfetchcmd("repo sync", d) os.chdir(codir) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.repo' --exclude '.git'" # Create a cache runfetchcmd( "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*")), d)
def __init__(self, d): self.cachedir = bb.data.getVar( "PERSISTENT_DIR", d, True) or bb.data.getVar("CACHE", d, True) if self.cachedir in [None, '']: bb.msg.fatal( bb.msg.domain.PersistData, "Please set the 'PERSISTENT_DIR' or 'CACHE' variable.") try: os.stat(self.cachedir) except OSError: bb.mkdirhier(self.cachedir) self.cachefile = os.path.join(self.cachedir, "bb_persist_data.sqlite3") bb.msg.debug( 1, bb.msg.domain.PersistData, "Using '%s' as the persistent data cache" % self.cachefile) self.connection = sqlite3.connect(self.cachefile, timeout=5, isolation_level=None)
def do_finish(): bb.mkdirhier(build_dir) os.chdir(pkgs_dir) global seen seen = [] for p in bb_get('PACKAGES').split(): pname = bb_get('NAME_%s' % p) print "Package: %s (%s)" % (p, pname) print "Description: ", bb_data['DESCRIPTION_'+p] files = bb_data['FILES_'+p].split(" ") pack_dir = pjoin(build_dir,p) bb.mkdirhier(pack_dir) install_files(files, pack_dir) for data in DATAS: if not bb_get(data+'_'+p): if data == 'NAME': bb_set(data+'_'+p, p) else: bb_set(data+'_'+p, bb_get(data+'_'+parent_pkg)) bb.mkdirhier(pjoin(pack_dir, 'CONTROL')) write_control_file(pjoin(pack_dir, 'CONTROL'), p)
def do_finish(): bb.mkdirhier(build_dir) os.chdir(pkgs_dir) global seen seen = [] for p in bb_get('PACKAGES').split(): pname = bb_get('NAME_%s' % p) print "Package: %s (%s)" % (p, pname) print "Description: ", bb_data['DESCRIPTION_' + p] files = bb_data['FILES_' + p].split(" ") pack_dir = pjoin(build_dir, p) bb.mkdirhier(pack_dir) install_files(files, pack_dir) for data in DATAS: if not bb_get(data + '_' + p): if data == 'NAME': bb_set(data + '_' + p, p) else: bb_set(data + '_' + p, bb_get(data + '_' + parent_pkg)) bb.mkdirhier(pjoin(pack_dir, 'CONTROL')) write_control_file(pjoin(pack_dir, 'CONTROL'), p)
def go(self, loc, ud, d): """Fetch url""" if os.access( os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir), '.bzr'), os.R_OK): bzrcmd = self._buildbzrcommand(ud, d, "update") logger.debug(1, "BZR Update %s", loc) os.chdir(os.path.join(ud.pkgdir, os.path.basename(ud.path))) runfetchcmd(bzrcmd, d) else: bb.utils.remove( os.path.join(ud.pkgdir, os.path.basename(ud.pkgdir)), True) bzrcmd = self._buildbzrcommand(ud, d, "fetch") logger.debug(1, "BZR Checkout %s", loc) bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", bzrcmd) runfetchcmd(bzrcmd, d) os.chdir(ud.pkgdir) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.bzr' --exclude '.bzrtags'" # tar them up to a defined filename try: runfetchcmd( "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(ud.pkgdir)), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def go(self, loc, ud, d): """Fetch url""" logger.debug( 2, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): svnupdatecmd = self._buildsvncommand(ud, d, "update") logger.info("Update " + loc) # update sources there os.chdir(ud.moddir) logger.debug(1, "Running %s", svnupdatecmd) runfetchcmd(svnupdatecmd, d) else: svnfetchcmd = self._buildsvncommand(ud, d, "fetch") logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) logger.debug(1, "Running %s", svnfetchcmd) runfetchcmd(svnfetchcmd, d) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude '.svn'" os.chdir(ud.pkgdir) # tar them up to a defined filename try: runfetchcmd( "tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def download(self, loc, ud, d): """Fetch url""" if ud.user: username = ud.user + '@' else: username = "" ud.repochanged = not os.path.exists(ud.fullmirror) # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.clonedir) and os.path.exists(ud.fullmirror): bb.mkdirhier(ud.clonedir) os.chdir(ud.clonedir) runfetchcmd("tar -xzf %s" % (ud.fullmirror), d) # If the repo still doesn't exist, fallback to cloning it if not os.path.exists(ud.clonedir): bb.fetch2.check_network_access(d, "git clone --bare %s%s" % (ud.host, ud.path)) runfetchcmd("%s clone --bare %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) os.chdir(ud.clonedir) # Update the checkout if needed needupdate = False for name in ud.names: if not self._contains_ref(ud.revisions[name], d): needupdate = True if needupdate: bb.fetch2.check_network_access(d, "git fetch %s%s" % (ud.host, ud.path)) try: runfetchcmd("%s remote prune origin" % ud.basecmd, d) runfetchcmd("%s remote rm origin" % ud.basecmd, d) except bb.fetch2.FetchError: logger.debug(1, "No Origin") runfetchcmd("%s remote add origin %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) runfetchcmd("%s fetch --all -t" % ud.basecmd, d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) ud.repochanged = True
def go(self, loc, ud, d): """Fetch url""" # try to use the tarball stash if Fetch.try_mirror(d, ud.localfile): bb.msg.debug( 1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping svn checkout." % ud.localpath ) return bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory '" + ud.moddir + "'") if os.access(os.path.join(ud.moddir, ".svn"), os.R_OK): svnupdatecmd = self._buildsvncommand(ud, d, "update") bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(ud.moddir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnupdatecmd) runfetchcmd(svnupdatecmd, d) else: svnfetchcmd = self._buildsvncommand(ud, d, "fetch") bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(ud.pkgdir) os.chdir(ud.pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % svnfetchcmd) runfetchcmd(svnfetchcmd, d) os.chdir(ud.pkgdir) # tar them up to a defined filename try: runfetchcmd("tar -czf %s %s" % (ud.localpath, ud.module), d) except: t, v, tb = sys.exc_info() try: os.unlink(ud.localpath) except OSError: pass raise t, v, tb
def install_files(files, root): import glob, errno, re, os for file in files: if os.path.isabs(file): file = '.' + file if not os.path.islink(file): if os.path.isdir(file): newfiles = [os.path.join(file, x) for x in os.listdir(file)] if newfiles: files += newfiles continue globbed = glob.glob(file) if globbed: if [file] != globbed: if not file in globbed: files += globbed continue else: globbed.remove(file) files += globbed if (not os.path.islink(file)) and (not os.path.exists(file)): continue if file[-4:] == '.pyc': continue if file in seen: continue seen.append(file) if os.path.isdir(file) and not os.path.islink(file): bb.mkdirhier(os.path.join(root, file)) os.chmod(os.path.join(root, file), os.stat(file).st_mode) continue fpath = os.path.join(root, file) dpath = os.path.dirname(fpath) bb.mkdirhier(dpath) ret = bb.copyfile(file, fpath) if ret is False: raise ("File population failed when copying %s to %s" % (file, fpath))
def install_files(files, root): import glob, errno, re,os for file in files: if os.path.isabs(file): file = '.' + file if not os.path.islink(file): if os.path.isdir(file): newfiles = [ os.path.join(file,x) for x in os.listdir(file) ] if newfiles: files += newfiles continue globbed = glob.glob(file) if globbed: if [ file ] != globbed: if not file in globbed: files += globbed continue else: globbed.remove(file) files += globbed if (not os.path.islink(file)) and (not os.path.exists(file)): continue if file[-4:] == '.pyc': continue if file in seen: continue seen.append(file) if os.path.isdir(file) and not os.path.islink(file): bb.mkdirhier(os.path.join(root,file)) os.chmod(os.path.join(root,file), os.stat(file).st_mode) continue fpath = os.path.join(root,file) dpath = os.path.dirname(fpath) bb.mkdirhier(dpath) ret = bb.copyfile(file, fpath) if ret is False: raise("File population failed when copying %s to %s" % (file, fpath))
def go(self, loc, ud, d): """Fetch url""" if os.access( os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK): bb.msg.debug( 1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping repo init / sync." % ud.localpath) return gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", ".")) repodir = data.getVar("REPODIR", d, True) or os.path.join( data.getVar("DL_DIR", d, True), "repo") codir = os.path.join(repodir, gitsrcname, ud.manifest) if ud.user: username = ud.user + "@" else: username = "" bb.mkdirhier(os.path.join(codir, "repo")) os.chdir(os.path.join(codir, "repo")) if not os.path.exists(os.path.join(codir, "repo", ".repo")): runfetchcmd( "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d) runfetchcmd("repo sync", d) os.chdir(codir) # Create a cache runfetchcmd( "tar --exclude=.repo --exclude=.git -czf %s %s" % (ud.localpath, os.path.join(".", "*")), d)
def go(self, loc, ud, d): # try to use the tarball stash if not self.forcefetch(loc, ud, d) and Fetch.try_mirror(d, ud.localfile): bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping cvs checkout." % ud.localpath) return method = "pserver" if "method" in ud.parm: method = ud.parm["method"] localdir = ud.module if "localdir" in ud.parm: localdir = ud.parm["localdir"] cvs_port = "" if "port" in ud.parm: cvs_port = ud.parm["port"] cvs_rsh = None if method == "ext": if "rsh" in ud.parm: cvs_rsh = ud.parm["rsh"] if method == "dir": cvsroot = ud.path else: cvsroot = ":" + method cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True) if cvsproxyhost: cvsroot += ";proxy=" + cvsproxyhost cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True) if cvsproxyport: cvsroot += ";proxyport=" + cvsproxyport cvsroot += ":" + ud.user if ud.pswd: cvsroot += ":" + ud.pswd cvsroot += "@" + ud.host + ":" + cvs_port + ud.path options = [] if 'norecurse' in ud.parm: options.append("-l") if ud.date: # treat YYYYMMDDHHMM specially for CVS if len(ud.date) == 12: options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12])) else: options.append("-D \"%s UTC\"" % ud.date) if ud.tag: options.append("-r %s" % ud.tag) localdata = data.createCopy(d) data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) data.update_data(localdata) data.setVar('CVSROOT', cvsroot, localdata) data.setVar('CVSCOOPTS', " ".join(options), localdata) data.setVar('CVSMODULE', ud.module, localdata) cvscmd = data.getVar('FETCHCOMMAND', localdata, 1) cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1) if cvs_rsh: cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) # create module directory bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: checking for module directory") pkg = data.expand('${PN}', d) pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg) moddir = os.path.join(pkgdir,localdir) if os.access(os.path.join(moddir,'CVS'), os.R_OK): bb.msg.note(1, bb.msg.domain.Fetcher, "Update " + loc) # update sources there os.chdir(moddir) myret = os.system(cvsupdatecmd) else: bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) # check out sources there bb.mkdirhier(pkgdir) os.chdir(pkgdir) bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s" % cvscmd) myret = os.system(cvscmd) if myret != 0 or not os.access(moddir, os.R_OK): try: os.rmdir(moddir) except OSError: pass raise FetchError(ud.module) # tar them up to a defined filename if 'fullpath' in ud.parm: os.chdir(pkgdir) myret = os.system("tar -czf %s %s" % (ud.localpath, localdir)) else: os.chdir(moddir) os.chdir('..') myret = os.system("tar -czf %s %s" % (ud.localpath, os.path.basename(moddir))) if myret != 0: try: os.unlink(ud.localpath) except OSError: pass raise FetchError(ud.module)
def go(self, loc, ud, d): """Fetch url""" if ud.user: username = ud.user + '@' else: username = "" repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) coname = '%s' % (ud.tag) codir = os.path.join(ud.clonedir, coname) if not os.path.exists(ud.clonedir): try: Fetch.try_mirrors(ud.mirrortarball) bb.mkdirhier(ud.clonedir) os.chdir(ud.clonedir) runfetchcmd("tar -xzf %s" % (repofile), d) except: runfetchcmd( "%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) os.chdir(ud.clonedir) # Remove all but the .git directory if not self._contains_ref(ud.tag, d): runfetchcmd("rm * -Rf", d) runfetchcmd( "%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d) runfetchcmd( "%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) os.chdir(ud.clonedir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) if mirror_tarballs != "0" or 'fullclone' in ud.parm: bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") runfetchcmd( "tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*")), d) if 'fullclone' in ud.parm: return if os.path.exists(codir): bb.utils.prunedir(codir) subdir = ud.parm.get("subpath", "") if subdir != "": if subdir.endswith("/"): subdirbase = os.path.basename(subdir[:-1]) else: subdirbase = os.path.basename(subdir) else: subdirbase = "" if subdir != "": readpathspec = ":%s" % (subdir) codir = os.path.join(codir, "git") coprefix = os.path.join(codir, subdirbase, "") else: readpathspec = "" coprefix = os.path.join(codir, "git", "") bb.mkdirhier(codir) os.chdir(ud.clonedir) runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) runfetchcmd( "%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d) os.chdir(codir) bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*")), d) os.chdir(ud.clonedir) bb.utils.prunedir(codir)
pkgs_dir = os.environ['packagingtmpdir'] print "searching for files in", pkgs_dir #where this script will build packages global build_dir build_dir = os.environ['IPKGBUILDDIR'] print "temporary files goes in", build_dir ####################################################################### ####################################################################### #Copied from bitbake (portage based) (c) import bb if not os.path.isdir(pkgs_dir): bb.mkdirhier(pkgs_dir) def install_files(files, root): import glob, errno, re, os for file in files: if os.path.isabs(file): file = '.' + file if not os.path.islink(file): if os.path.isdir(file): newfiles = [os.path.join(file, x) for x in os.listdir(file)] if newfiles: files += newfiles continue globbed = glob.glob(file)
def go(self, loc, ud, d): """Fetch url""" if Fetch.try_mirror(d, ud.localfile): bb.msg.debug( 1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath) return if ud.user: username = ud.user + '@' else: username = "" gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.')) repofilename = 'git_%s.tar.gz' % (gitsrcname) repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename) repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname) coname = '%s' % (ud.tag) if ud.subdir != "": coname = '%s_%s' % (coname, ud.subdir.replace('/', '.')) codir = os.path.join(repodir, coname) """ A single repodir can be used for multiple checkouts. Protect against corruption. """ lf = bb.utils.lockfile("%s.%s" % (repofile, '.lock')) if not os.path.exists(repodir): if Fetch.try_mirror(d, repofilename): bb.mkdirhier(repodir) os.chdir(repodir) runfetchcmd("tar -xzf %s" % (repofile), d) else: runfetchcmd( "git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d) os.chdir(repodir) # Remove all but the .git directory if not self._contains_ref(ud.tag, d): runfetchcmd("rm * -Rf", d) runfetchcmd( "git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d) runfetchcmd( "git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d) runfetchcmd("git prune-packed", d) runfetchcmd("git pack-redundant --all | xargs -r rm", d) os.chdir(repodir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) if mirror_tarballs != "0": bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository") runfetchcmd( "tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*")), d) if os.path.exists(codir): bb.utils.prunedir(codir) if ud.subdir != "": readpathspec = ":%s" % (ud.subdir) subdir = os.path.basename(ud.subdir) else: readpathspec = "" subdir = "git" bb.mkdirhier(codir) os.chdir(repodir) runfetchcmd("git read-tree %s%s" % (ud.tag, readpathspec), d) runfetchcmd( "git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, subdir, "")), d) count = runfetchcmd("git rev-list %s -- | wc -l" % (ud.tag), d, True) bb.utils.unlockfile(lf) os.chdir(codir) bb.msg.note(1, bb.msg.domain.Fetcher, "Checkins count: %s" % count) f = open(os.path.join(subdir, '.git_revision_count'), 'w') f.write(count) f.close() bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout") runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*")), d) os.chdir(repodir) bb.utils.prunedir(codir)
def go(self, loc, ud, d): """Fetch url""" if ud.user: username = ud.user + '@' else: username = "" repofile = os.path.join(data.getVar("DL_DIR", d, 1), ud.mirrortarball) coname = '%s' % (ud.tag) codir = os.path.join(ud.clonedir, coname) # If we have no existing clone and no mirror tarball, try and obtain one if not os.path.exists(ud.clonedir) and not os.path.exists(repofile): try: Fetch.try_mirrors(ud.mirrortarball) except: pass # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.clonedir) and os.path.exists(repofile): bb.mkdirhier(ud.clonedir) os.chdir(ud.clonedir) runfetchcmd("tar -xzf %s" % (repofile), d) # If the repo still doesn't exist, fallback to cloning it if not os.path.exists(ud.clonedir): runfetchcmd("%s clone -n %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.clonedir), d) os.chdir(ud.clonedir) # Update the checkout if needed if not self._contains_ref(ud.tag, d) or 'fullclone' in ud.parm: # Remove all but the .git directory runfetchcmd("rm * -Rf", d) if 'fullclone' in ud.parm: runfetchcmd("%s fetch --all" % (ud.basecmd), d) else: runfetchcmd("%s fetch %s://%s%s%s %s" % (ud.basecmd, ud.proto, username, ud.host, ud.path, ud.branch), d) runfetchcmd("%s fetch --tags %s://%s%s%s" % (ud.basecmd, ud.proto, username, ud.host, ud.path), d) runfetchcmd("%s prune-packed" % ud.basecmd, d) runfetchcmd("%s pack-redundant --all | xargs -r rm" % ud.basecmd, d) # Generate a mirror tarball if needed os.chdir(ud.clonedir) mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) if mirror_tarballs != "0" or 'fullclone' in ud.parm: logger.info("Creating tarball of git repository") runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d) if 'fullclone' in ud.parm: return if os.path.exists(codir): bb.utils.prunedir(codir) subdir = ud.parm.get("subpath", "") if subdir != "": if subdir.endswith("/"): subdirbase = os.path.basename(subdir[:-1]) else: subdirbase = os.path.basename(subdir) else: subdirbase = "" if subdir != "": readpathspec = ":%s" % (subdir) codir = os.path.join(codir, "git") coprefix = os.path.join(codir, subdirbase, "") else: readpathspec = "" coprefix = os.path.join(codir, "git", "") scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": runfetchcmd("%s clone -n %s %s" % (ud.basecmd, ud.clonedir, coprefix), d) os.chdir(coprefix) runfetchcmd("%s checkout -q -f %s%s" % (ud.basecmd, ud.tag, readpathspec), d) else: bb.mkdirhier(codir) os.chdir(ud.clonedir) runfetchcmd("%s read-tree %s%s" % (ud.basecmd, ud.tag, readpathspec), d) runfetchcmd("%s checkout-index -q -f --prefix=%s -a" % (ud.basecmd, coprefix), d) os.chdir(codir) logger.info("Creating tarball of git checkout") runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d) os.chdir(ud.clonedir) bb.utils.prunedir(codir)
def go(self, loc, ud, d): method = ud.parm.get('method', 'pserver') localdir = ud.parm.get('localdir', ud.module) cvs_port = ud.parm.get('port', '') cvs_rsh = None if method == "ext": if "rsh" in ud.parm: cvs_rsh = ud.parm["rsh"] if method == "dir": cvsroot = ud.path else: cvsroot = ":" + method cvsproxyhost = data.getVar('CVS_PROXY_HOST', d, True) if cvsproxyhost: cvsroot += ";proxy=" + cvsproxyhost cvsproxyport = data.getVar('CVS_PROXY_PORT', d, True) if cvsproxyport: cvsroot += ";proxyport=" + cvsproxyport cvsroot += ":" + ud.user if ud.pswd: cvsroot += ":" + ud.pswd cvsroot += "@" + ud.host + ":" + cvs_port + ud.path options = [] if 'norecurse' in ud.parm: options.append("-l") if ud.date: # treat YYYYMMDDHHMM specially for CVS if len(ud.date) == 12: options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12])) else: options.append("-D \"%s UTC\"" % ud.date) if ud.tag: options.append("-r %s" % ud.tag) localdata = data.createCopy(d) data.setVar('OVERRIDES', "cvs:%s" % data.getVar('OVERRIDES', localdata), localdata) data.update_data(localdata) data.setVar('CVSROOT', cvsroot, localdata) data.setVar('CVSCOOPTS', " ".join(options), localdata) data.setVar('CVSMODULE', ud.module, localdata) cvscmd = data.getVar('FETCHCOMMAND', localdata, 1) cvsupdatecmd = data.getVar('UPDATECOMMAND', localdata, 1) if cvs_rsh: cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) # create module directory logger.debug(2, "Fetch: checking for module directory") pkg = data.expand('${PN}', d) pkgdir = os.path.join(data.expand('${CVSDIR}', localdata), pkg) moddir = os.path.join(pkgdir, localdir) if os.access(os.path.join(moddir, 'CVS'), os.R_OK): logger.info("Update " + loc) # update sources there os.chdir(moddir) myret = os.system(cvsupdatecmd) else: logger.info("Fetch " + loc) # check out sources there bb.mkdirhier(pkgdir) os.chdir(pkgdir) logger.debug(1, "Running %s", cvscmd) myret = os.system(cvscmd) if myret != 0 or not os.access(moddir, os.R_OK): try: os.rmdir(moddir) except OSError: pass raise FetchError(ud.module) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude 'CVS'" # tar them up to a defined filename if 'fullpath' in ud.parm: os.chdir(pkgdir) myret = os.system("tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir)) else: os.chdir(moddir) os.chdir('..') myret = os.system( "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir))) if myret != 0: try: os.unlink(ud.localpath) except OSError: pass raise FetchError(ud.module)
def exec_func(func, d, dirs=None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: return flags = data.getVarFlags(func, d) for item in [ 'deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot' ]: if not item in flags: flags[item] = None ispython = flags['python'] cleandirs = (data.expand(flags['cleandirs'], d) or "").split() for cdir in cleandirs: os.system("rm -rf %s" % cdir) if dirs: dirs = data.expand(dirs, d) else: dirs = (data.expand(flags['dirs'], d) or "").split() for adir in dirs: mkdirhier(adir) if len(dirs) > 0: adir = dirs[-1] else: adir = data.getVar('B', d, 1) # Save current directory try: prevdir = os.getcwd() except OSError: prevdir = data.getVar('TOPDIR', d, True) # Setup logfiles t = data.getVar('T', d, 1) if not t: bb.msg.fatal(bb.msg.domain.Build, "T not set") mkdirhier(t) logfile = "%s/log.%s.%s" % (t, func, str(os.getpid())) runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) # Change to correct directory (if specified) if adir and os.access(adir, os.F_OK): os.chdir(adir) # Handle logfiles si = file('/dev/null', 'r') try: if bb.msg.debug_level['default'] > 0 or ispython: so = os.popen("tee \"%s\"" % logfile, "w") else: so = file(logfile, 'w') except OSError, e: bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) pass
def go(self, loc, ud, d): """ Fetch urls """ # try to use the tarball stash if Fetch.try_mirror(d, ud.localfile): bb.msg.debug( 1, bb.msg.domain.Fetcher, "%s already exists or was mirrored, skipping perforce checkout." % ud.localpath) return (host, depot, user, pswd, parm) = Perforce.doparse(loc, d) if depot.find('/...') != -1: path = depot[:depot.find('/...')] else: path = depot if "module" in parm: module = parm["module"] else: module = os.path.basename(path) localdata = data.createCopy(d) data.setVar('OVERRIDES', "p4:%s" % data.getVar('OVERRIDES', localdata), localdata) data.update_data(localdata) # Get the p4 command p4opt = "" if user: p4opt += " -u %s" % (user) if pswd: p4opt += " -P %s" % (pswd) if host: p4opt += " -p %s" % (host) p4cmd = data.getVar('FETCHCOMMAND', localdata, 1) # create temp directory bb.msg.debug(2, bb.msg.domain.Fetcher, "Fetch: creating temporary directory") bb.mkdirhier(data.expand('${WORKDIR}', localdata)) data.setVar('TMPBASE', data.expand('${WORKDIR}/oep4.XXXXXX', localdata), localdata) tmppipe = os.popen( data.getVar('MKTEMPDIRCMD', localdata, 1) or "false") tmpfile = tmppipe.readline().strip() if not tmpfile: bb.error( "Fetch: unable to create temporary directory.. make sure 'mktemp' is in the PATH." ) raise FetchError(module) if "label" in parm: depot = "%s@%s" % (depot, parm["label"]) else: cset = Perforce.getcset(d, depot, host, user, pswd, parm) depot = "%s@%s" % (depot, cset) os.chdir(tmpfile) bb.msg.note(1, bb.msg.domain.Fetcher, "Fetch " + loc) bb.msg.note(1, bb.msg.domain.Fetcher, "%s%s files %s" % (p4cmd, p4opt, depot)) p4file = os.popen("%s%s files %s" % (p4cmd, p4opt, depot)) if not p4file: bb.error("Fetch: unable to get the P4 files from %s" % (depot)) raise FetchError(module) count = 0 for file in p4file: list = file.split() if list[2] == "delete": continue dest = list[0][len(path) + 1:] where = dest.find("#") os.system("%s%s print -o %s/%s %s" % (p4cmd, p4opt, module, dest[:where], list[0])) count = count + 1 if count == 0: bb.error("Fetch: No files gathered from the P4 fetch") raise FetchError(module) myret = os.system("tar -czf %s %s" % (ud.localpath, module)) if myret != 0: try: os.unlink(ud.localpath) except OSError: pass raise FetchError(module) # cleanup os.system('rm -rf %s' % tmpfile)
def exec_func(func, d, dirs = None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: return flags = data.getVarFlags(func, d) for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']: if not item in flags: flags[item] = None ispython = flags['python'] cleandirs = (data.expand(flags['cleandirs'], d) or "").split() for cdir in cleandirs: os.system("rm -rf %s" % cdir) if dirs: dirs = data.expand(dirs, d) else: dirs = (data.expand(flags['dirs'], d) or "").split() for adir in dirs: mkdirhier(adir) if len(dirs) > 0: adir = dirs[-1] else: adir = data.getVar('B', d, 1) # Save current directory try: prevdir = os.getcwd() except OSError: prevdir = data.getVar('TOPDIR', d, True) # Setup logfiles t = data.getVar('T', d, 1) if not t: bb.msg.fatal(bb.msg.domain.Build, "T not set") mkdirhier(t) # Gross hack, FIXME import random logfile = "%s/log.%s.%s.%s" % (t, func, str(os.getpid()),random.random()) runfile = "%s/run.%s.%s" % (t, func, str(os.getpid())) # Change to correct directory (if specified) if adir and os.access(adir, os.F_OK): os.chdir(adir) # Handle logfiles si = file('/dev/null', 'r') try: if bb.msg.debug_level['default'] > 0 or ispython: so = os.popen("tee \"%s\"" % logfile, "w") else: so = file(logfile, 'w') except OSError, e: bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e) pass
def exec_func(func, d, dirs = None): """Execute an BB 'function'""" body = data.getVar(func, d) if not body: return flags = data.getVarFlags(func, d) for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']: if not item in flags: flags[item] = None ispython = flags['python'] cleandirs = (data.expand(flags['cleandirs'], d) or "").split() for cdir in cleandirs: os.system("rm -rf %s" % cdir) if dirs: dirs = data.expand(dirs, d) else: dirs = (data.expand(flags['dirs'], d) or "").split() for adir in dirs: mkdirhier(adir) if len(dirs) > 0: adir = dirs[-1] else: adir = data.getVar('B', d, 1) try: prevdir = os.getcwd() except OSError: prevdir = data.getVar('TOPDIR', d, True) if adir and os.access(adir, os.F_OK): os.chdir(adir) locks = [] lockfiles = (data.expand(flags['lockfiles'], d) or "").split() for lock in lockfiles: locks.append(bb.utils.lockfile(lock)) if flags['python']: exec_func_python(func, d) else: exec_func_shell(func, d, flags) for lock in locks: bb.utils.unlockfile(lock) if os.path.exists(prevdir): os.chdir(prevdir)