def fetch_uri(uri, ud, d):
if checkonly:
fetchcmd = data.getVar("CHECKCOMMAND", d, True)
elif os.path.exists(ud.localpath):
# file exists, but we didnt complete it.. trying again..
fetchcmd = data.getVar("RESUMECOMMAND", d, True)
else:
fetchcmd = data.getVar("FETCHCOMMAND", d, True)
uri = uri.split(";")[0]
uri_decoded = list(decodeurl(uri))
uri_type = uri_decoded[0]
uri_host = uri_decoded[1]
fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0])
fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
if not checkonly:
logger.info("fetch " + uri)
logger.debug(2, "executing " + fetchcmd)
bb.fetch2.check_network_access(d, fetchcmd)
runfetchcmd(fetchcmd, d, quiet=checkonly)
# Sanity check since wget can pretend it succeed when it didn't
# Also, this used to happen if sourceforge sent us to the mirror page
if not os.path.exists(ud.localpath) and not checkonly:
raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
def stamp_is_current(task, d, checkdeps = 1):
"""Check status of a given task's stamp. returns 0 if it is not current and needs updating."""
task_graph = data.getVar('_task_graph', d)
if not task_graph:
task_graph = bb.digraph()
data.setVar('_task_graph', task_graph, d)
stamp = data.getVar('STAMP', d)
if not stamp:
return 0
stampfile = "%s.%s" % (data.expand(stamp, d), task)
if not os.access(stampfile, os.F_OK):
return 0
if checkdeps == 0:
return 1
import stat
tasktime = os.stat(stampfile)[stat.ST_MTIME]
_deps = []
def checkStamp(graph, task):
# check for existance
if data.getVarFlag(task, 'nostamp', d):
return 1
if not stamp_is_current(task, d, 0):
return 0
depfile = "%s.%s" % (data.expand(stamp, d), task)
deptime = os.stat(depfile)[stat.ST_MTIME]
if deptime > tasktime:
return 0
return 1
return task_graph.walkdown(task, checkStamp)
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
if not dirs:
dirs = (data.getVarFlag(func, 'dirs', d) or "").split()
for adir in dirs:
adir = data.expand(adir, d)
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
adir = data.expand(adir, d)
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.expand('${TOPDIR}', d)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
if data.getVarFlag(func, "python", d):
exec_func_python(func, d)
else:
exec_func_shell(func, d)
os.chdir(prevdir)
def fetch_uri(uri, ud, d):
if os.path.exists(ud.localpath):
# file exists, but we didnt complete it.. trying again..
fetchcmd = data.getVar("RESUMECOMMAND", d, 1)
else:
fetchcmd = data.getVar("FETCHCOMMAND", d, 1)
bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
fetchcmd = fetchcmd.replace("${URI}", uri)
fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
bb.msg.debug(2, bb.msg.domain.Fetcher, "executing " + fetchcmd)
ret = os.system(fetchcmd)
if ret != 0:
return False
# Sanity check since wget can pretend it succeed when it didn't
# Also, this used to happen if sourceforge sent us to the mirror page
if not os.path.exists(ud.localpath):
bb.msg.debug(
2,
bb.msg.domain.Fetcher,
"The fetch command for %s returned success but %s doesn't exist?..." % (uri, ud.localpath),
)
return False
return True
def try_mirror(d, tarfn):
"""
Try to use a mirrored version of the sources. We do this
to avoid massive loads on foreign cvs and svn servers.
This method will be used by the different fetcher
implementations.
d Is a bb.data instance
tarfn is the name of the tarball
"""
tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
if os.access(tarpath, os.R_OK):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists, skipping checkout." % tarfn)
return True
pn = data.getVar('PN', d, True)
src_tarball_stash = None
if pn:
src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
for stash in src_tarball_stash:
fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
uri = stash + tarfn
bb.msg.note(1, bb.msg.domain.Fetcher, "fetch " + uri)
fetchcmd = fetchcmd.replace("${URI}", uri)
ret = os.system(fetchcmd)
if ret == 0:
bb.msg.note(1, bb.msg.domain.Fetcher, "Fetched %s from tarball stash, skipping checkout" % tarfn)
return True
return False
def download(self, loc, ud, d):
"""
do fetch
"""
# if the package has been downloaded, just return
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
logger.debug(1, "%s already exists (or was stashed). Skipping gclient sync.", ud.localpath)
return
depot_dir = data.getVar("DEPOTDIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "depot")
sync_dir = os.path.join(depot_dir, ud.packname)
bb.utils.mkdirhier(sync_dir)
os.chdir(sync_dir)
if not os.path.exists(os.path.join(sync_dir, ".gclient")):
logger.info('This is the first time to sync this depot, config it as htttp://%s%s'
% (ud.host, ud.path))
runfetchcmd('gclient config http://%s%s' % (ud.host, ud.path), d)
logger.info('Start to sync source code..')
runfetchcmd('gclient fetch --jobs %s' % ud.njobs, d)
logger.info('Creating tarball %s.' % ud.localfile)
runfetchcmd('tar --exclude .svn --exclude .git -czf %s ./' %
os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), d)
def download(self, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.utils.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), ud.url)
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url, ud.url)
runfetchcmd("repo sync", d)
os.chdir(codir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude='.repo' --exclude='.git'"
# Create a cache
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)
def localpaths(self, urldata, d):
"""
Return the local filename of a given url assuming a successful fetch.
"""
searched = []
path = urldata.decodedurl
newpath = path
if path[0] == "/":
return [path]
filespath = data.getVar('FILESPATH', d, True)
if filespath:
logger.debug(2, "Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":"))))
newpath, hist = bb.utils.which(filespath, path, history=True)
searched.extend(hist)
if not newpath:
filesdir = data.getVar('FILESDIR', d, True)
if filesdir:
logger.debug(2, "Searching for %s in path: %s" % (path, filesdir))
newpath = os.path.join(filesdir, path)
searched.append(newpath)
if (not newpath or not os.path.exists(newpath)) and path.find("*") != -1:
# For expressions using '*', best we can do is take the first directory in FILESPATH that exists
newpath, hist = bb.utils.which(filespath, ".", history=True)
searched.extend(hist)
logger.debug(2, "Searching for %s in path: %s" % (path, newpath))
return searched
if not os.path.exists(newpath):
dldirfile = os.path.join(d.getVar("DL_DIR", True), path)
logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
bb.utils.mkdirhier(os.path.dirname(dldirfile))
searched.append(dldirfile)
return searched
return searched
def __init__(self, data):
self.basehash = {}
self.taskhash = {}
self.unihash = {}
self.taskdeps = {}
self.runtaskdeps = {}
self.file_checksum_values = {}
self.taints = {}
self.gendeps = {}
self.lookupcache = {}
self.setscenetasks = set()
self.basehash_ignore_vars = set((data.getVar("BB_BASEHASH_IGNORE_VARS")
or "").split())
self.taskhash_ignore_tasks = None
self.init_rundepcheck(data)
checksum_cache_file = data.getVar("BB_HASH_CHECKSUM_CACHE_FILE")
if checksum_cache_file:
self.checksum_cache = FileChecksumCache()
self.checksum_cache.init_cache(data, checksum_cache_file)
else:
self.checksum_cache = None
self.unihash_cache = bb.cache.SimpleCache("3")
self.unitaskhashes = self.unihash_cache.init_cache(
data, "bb_unihashes.dat", {})
self.localdirsexclude = (
data.getVar("BB_SIGNATURE_LOCAL_DIRS_EXCLUDE")
or "CVS .bzr .git .hg .osc .p4 .repo .svn").split()
self.tidtopn = {}
def localpath(self, url, urldata, d):
"""
Return the local filename of a given url assuming a successful fetch.
"""
path = urldata.decodedurl
newpath = path
if path[0] != "/":
filespath = data.getVar('FILESPATH', d, True)
if filespath:
logger.debug(
2, "Searching for %s in paths: \n%s" %
(path, "\n ".join(filespath.split(":"))))
newpath = bb.utils.which(filespath, path)
if not newpath:
filesdir = data.getVar('FILESDIR', d, True)
if filesdir:
logger.debug(
2, "Searching for %s in path: %s" % (path, filesdir))
newpath = os.path.join(filesdir, path)
if not newpath or not os.path.exists(
newpath) and path.find("*") != -1:
# For expressions using '*', best we can do is take the first directory in FILESPATH that exists
newpath = bb.utils.which(filespath, ".")
logger.debug(2,
"Searching for %s in path: %s" % (path, newpath))
return newpath
if not os.path.exists(newpath):
dldirfile = os.path.join(d.getVar("DL_DIR", True), path)
logger.debug(2, "Defaulting to %s for %s" % (dldirfile, path))
bb.utils.mkdirhier(os.path.dirname(dldirfile))
return dldirfile
return newpath
def getcset(d, depot, host, user, pswd, parm):
p4opt = ""
if "cset" in parm:
return parm["cset"]
if user:
p4opt += " -u %s" % (user)
if pswd:
p4opt += " -P %s" % (pswd)
if host:
p4opt += " -p %s" % (host)
p4date = data.getVar("P4DATE", d, 1)
if "revision" in parm:
depot += "#%s" % (parm["revision"])
elif "label" in parm:
depot += "@%s" % (parm["label"])
elif p4date:
depot += "@%s" % (p4date)
p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1)
bb.msg.debug(1, bb.msg.domain.Fetcher,
"Running %s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
p4file = os.popen("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
cset = p4file.readline().strip()
bb.msg.debug(1, bb.msg.domain.Fetcher, "READ %s" % (cset))
if not cset:
return -1
return cset.split(' ')[1]
def getcset(d, depot,host,user,pswd,parm):
if "cset" in parm:
return parm["cset"];
if user:
data.setVar('P4USER', user, d)
if pswd:
data.setVar('P4PASSWD', pswd, d)
if host:
data.setVar('P4PORT', host, d)
p4date = data.getVar("P4DATE", d, 1)
if "revision" in parm:
depot += "#%s" % (parm["revision"])
elif "label" in parm:
depot += "@%s" % (parm["label"])
elif p4date:
depot += "@%s" % (p4date)
p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s changes -m 1 %s" % (p4cmd, depot))
p4file = os.popen("%s changes -m 1 %s" % (p4cmd,depot))
cset = p4file.readline().strip()
bb.msg.debug(1, bb.msg.domain.Fetcher, "READ %s" % (cset))
if not cset:
return -1
return cset.split(' ')[1]
def load_bbfile(bbfile, appends, config):
"""
Load and parse one .bb build file
Return the data and whether parsing resulted in the file being skipped
"""
chdir_back = False
from bb import data, parse
# expand tmpdir to include this topdir
data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
oldpath = os.path.abspath(os.getcwd())
parse.cached_mtime_noerror(bbfile_loc)
bb_data = data.init_db(config)
# The ConfHandler first looks if there is a TOPDIR and if not
# then it would call getcwd().
# Previously, we chdir()ed to bbfile_loc, called the handler
# and finally chdir()ed back, a couple of thousand times. We now
# just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
if not data.getVar('TOPDIR', bb_data):
chdir_back = True
data.setVar('TOPDIR', bbfile_loc, bb_data)
try:
if appends:
data.setVar('__BBAPPEND', " ".join(appends), bb_data)
bb_data = parse.handle(bbfile, bb_data)
if chdir_back:
os.chdir(oldpath)
return bb_data
except:
if chdir_back:
os.chdir(oldpath)
raise
def srcrev_internal_helper(ud, d):
"""
Return:
a) a source revision if specified
b) True if auto srcrev is in action
c) False otherwise
"""
if 'rev' in ud.parm:
return ud.parm['rev']
if 'tag' in ud.parm:
return ud.parm['tag']
rev = None
if 'name' in ud.parm:
pn = data.getVar("PN", d, 1)
rev = data.getVar("SRCREV_%s_pn-%s" % (ud.parm['name'], pn), d, 1)
if not rev:
rev = data.getVar("SRCREV_pn-%s_%s" % (pn, ud.parm['name']), d,
1)
if not rev:
rev = data.getVar("SRCREV_%s" % (ud.parm['name']), d, 1)
if not rev:
rev = data.getVar("SRCREV", d, 1)
if rev == "INVALID":
raise InvalidSRCREV("Please set SRCREV to a valid value")
if not rev:
return False
if rev == "SRCREVINACTION":
return True
return rev
def srcrev_internal_helper(ud, d):
"""
Return:
a) a source revision if specified
b) True if auto srcrev is in action
c) False otherwise
"""
if 'rev' in ud.parm:
return ud.parm['rev']
if 'tag' in ud.parm:
return ud.parm['tag']
rev = None
if 'name' in ud.parm:
pn = data.getVar("PN", d, 1)
rev = data.getVar("SRCREV_pn-" + pn + "_" + ud.parm['name'], d, 1)
if not rev:
rev = data.getVar("SRCREV", d, 1)
if not rev:
return False
if rev is "SRCREVINACTION":
return True
return rev
def localpath(self, url, ud, d):
if not "module" in ud.parm:
raise MissingParameterError("osc method needs a 'module' parameter.")
ud.module = ud.parm["module"]
# Create paths to osc checkouts
relpath = ud.path
if relpath.startswith('/'):
# Remove leading slash as os.path.join can't cope
relpath = relpath[1:]
ud.pkgdir = os.path.join(data.expand('${OSCDIR}', d), ud.host)
ud.moddir = os.path.join(ud.pkgdir, relpath, ud.module)
if 'rev' in ud.parm:
ud.revision = ud.parm['rev']
else:
pv = data.getVar("PV", d, 0)
rev = Fetch.srcrev_internal_helper(ud, d)
if rev and rev != True:
ud.revision = rev
else:
ud.revision = ""
ud.localfile = data.expand('%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.path.replace('/', '.'), ud.revision), d)
return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
runfetchcmd("repo sync", d)
os.chdir(codir)
# Create a cache
runfetchcmd("tar --exclude=.repo --exclude=.git -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
def download(self, uri, ud, d, checkonly = False):
"""Fetch urls"""
if checkonly:
fetchcmd = data.getVar("CHECKCOMMAND_wget", d, True) or d.expand("/usr/bin/env wget --spider -t 5 --passive-ftp --no-check-certificate -P ${DL_DIR} '${URI}'")
elif os.path.exists(ud.localpath):
# file exists, but we didnt complete it.. trying again..
fetchcmd = data.getVar("RESUMECOMMAND_wget", d, True) or d.expand("/usr/bin/env wget -c -t 5 -nv --passive-ftp --no-check-certificate -P ${DL_DIR} '${URI}'")
else:
fetchcmd = data.getVar("FETCHCOMMAND_wget", d, True) or d.expand("/usr/bin/env wget -t 5 -nv --passive-ftp --no-check-certificate -P ${DL_DIR} '${URI}'")
uri = uri.split(";")[0]
uri_decoded = list(decodeurl(uri))
uri_type = uri_decoded[0]
uri_host = uri_decoded[1]
fetchcmd = fetchcmd.replace("${URI}", uri.split(";")[0])
fetchcmd = fetchcmd.replace("${FILE}", ud.basename)
if not checkonly:
logger.info("fetch " + uri)
logger.debug(2, "executing " + fetchcmd)
bb.fetch2.check_network_access(d, fetchcmd)
runfetchcmd(fetchcmd, d, quiet=checkonly)
# Sanity check since wget can pretend it succeed when it didn't
# Also, this used to happen if sourceforge sent us to the mirror page
if not os.path.exists(ud.localpath) and not checkonly:
raise FetchError("The fetch command returned success for url %s but %s doesn't exist?!" % (uri, ud.localpath), uri)
return True
def getcset(d, depot,host,user,pswd,parm):
if "cset" in parm:
return parm["cset"];
if user:
data.setVar('P4USER', user, d)
if pswd:
data.setVar('P4PASSWD', pswd, d)
if host:
data.setVar('P4PORT', host, d)
p4date = data.getVar("P4DATE", d, 1)
if "revision" in parm:
depot += "#%s" % (parm["revision"])
elif "label" in parm:
depot += "@%s" % (parm["label"])
elif p4date:
depot += "@%s" % (p4date)
p4cmd = data.getVar('FETCHCOMMAND_p4', d, 1)
bb.msg.debug(1, bb.msg.domain.Fetcher, "Running %s changes -m 1 %s" % (p4cmd, depot))
p4file = os.popen("%s changes -m 1 %s" % (p4cmd,depot))
cset = p4file.readline().strip()
bb.msg.debug(1, bb.msg.domain.Fetcher, "READ %s" % (cset))
if not cset:
return -1
return cset.split(' ')[1]
def download(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
logger.debug(1, "%s already exists (or was stashed). Skipping repo init / sync.", ud.localpath)
return
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
repodir = data.getVar("REPODIR", d, True) or os.path.join(data.getVar("DL_DIR", d, True), "repo")
codir = os.path.join(repodir, gitsrcname, ud.manifest)
if ud.user:
username = ud.user + "@"
else:
username = ""
bb.mkdirhier(os.path.join(codir, "repo"))
os.chdir(os.path.join(codir, "repo"))
if not os.path.exists(os.path.join(codir, "repo", ".repo")):
bb.fetch2.check_network_access(d, "repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path))
runfetchcmd("repo init -m %s -b %s -u %s://%s%s%s" % (ud.manifest, ud.branch, ud.proto, username, ud.host, ud.path), d)
bb.fetch2.check_network_access(d, "repo sync %s" % ud.url)
runfetchcmd("repo sync", d)
os.chdir(codir)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude '.repo' --exclude '.git'"
# Create a cache
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.join(".", "*") ), d)
def add_task(task, deps, d):
task_graph = data.getVar('_task_graph', d)
if not task_graph:
task_graph = bb.digraph()
data.setVarFlag(task, 'task', 1, d)
task_graph.addnode(task, None)
for dep in deps:
if not task_graph.hasnode(dep):
task_graph.addnode(dep, None)
task_graph.addnode(task, dep)
# don't assume holding a reference
data.setVar('_task_graph', task_graph, d)
task_deps = data.getVar('_task_deps', d)
if not task_deps:
task_deps = {}
def getTask(name):
deptask = data.getVarFlag(task, name, d)
if deptask:
if not name in task_deps:
task_deps[name] = {}
task_deps[name][task] = deptask
getTask('deptask')
getTask('rdeptask')
getTask('recrdeptask')
getTask('nostamp')
data.setVar('_task_deps', task_deps, d)
def md5_sum(basename, d):
"""
Fast and incomplete OVERRIDE implementation for MD5SUM handling
MD5SUM_basename = "SUM" and fallback to MD5SUM_basename
"""
var = "MD5SUM_%s" % basename
return data.getVar(var, d) or data.getVar("MD5SUM", d)
def srcrev_internal_helper(ud, d):
"""
Return:
a) a source revision if specified
b) True if auto srcrev is in action
c) False otherwise
"""
if 'rev' in ud.parm:
return ud.parm['rev']
if 'tag' in ud.parm:
return ud.parm['tag']
rev = None
if 'name' in ud.parm:
pn = data.getVar("PN", d, 1)
rev = data.getVar("SRCREV_pn-" + pn + "_" + ud.parm['name'], d, 1)
if not rev:
rev = data.getVar("SRCREV", d, 1)
if rev == "INVALID":
raise InvalidSRCREV("Please set SRCREV to a valid value")
if not rev:
return False
if rev is "SRCREVINACTION":
return True
return rev
def load_bbfile(bbfile, appends, config):
"""
Load and parse one .bb build file
Return the data and whether parsing resulted in the file being skipped
"""
chdir_back = False
from bb import data, parse
# expand tmpdir to include this topdir
data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
oldpath = os.path.abspath(os.getcwd())
parse.cached_mtime_noerror(bbfile_loc)
bb_data = data.init_db(config)
# The ConfHandler first looks if there is a TOPDIR and if not
# then it would call getcwd().
# Previously, we chdir()ed to bbfile_loc, called the handler
# and finally chdir()ed back, a couple of thousand times. We now
# just fill in TOPDIR to point to bbfile_loc if there is no TOPDIR yet.
if not data.getVar('TOPDIR', bb_data):
chdir_back = True
data.setVar('TOPDIR', bbfile_loc, bb_data)
try:
if appends:
data.setVar('__BBAPPEND', " ".join(appends), bb_data)
bb_data = parse.handle(bbfile, bb_data)
if chdir_back:
os.chdir(oldpath)
return bb_data
except:
if chdir_back:
os.chdir(oldpath)
raise
def getcset(d, depot, host, user, pswd, parm):
p4opt = ""
if "cset" in parm:
return parm["cset"];
if user:
p4opt += " -u %s" % (user)
if pswd:
p4opt += " -P %s" % (pswd)
if host:
p4opt += " -p %s" % (host)
p4date = data.getVar("P4DATE", d, True)
if "revision" in parm:
depot += "#%s" % (parm["revision"])
elif "label" in parm:
depot += "@%s" % (parm["label"])
elif p4date:
depot += "@%s" % (p4date)
p4cmd = data.getVar('FETCHCOMMAND_p4', d, True)
logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot)
p4file, errors = bb.process.run("%s%s changes -m 1 %s" % (p4cmd, p4opt, depot))
cset = p4file.strip()
logger.debug(1, "READ %s", cset)
if not cset:
return -1
return cset.split(' ')[1]
def getSRCDate(d):
"""
Return the SRC Date for the component
d the bb.data module
"""
return data.getVar("SRCDATE", d, 1) or data.getVar("CVSDATE", d, 1) or data.getVar("DATE", d, 1 )
def srcrev_internal_helper(ud, d, name):
"""
Return:
a) a source revision if specified
b) latest revision if SRCREV="AUTOINC"
c) None if not specified
"""
if 'rev' in ud.parm:
return ud.parm['rev']
if 'tag' in ud.parm:
return ud.parm['tag']
rev = None
if name != '':
pn = data.getVar("PN", d, True)
rev = data.getVar("SRCREV_%s_pn-%s" % (name, pn), d, True)
if not rev:
rev = data.getVar("SRCREV_%s" % name, d, True)
if not rev:
rev = data.getVar("SRCREV", d, True)
if rev == "INVALID":
raise FetchError("Please set SRCREV to a valid value", ud.url)
if rev == "AUTOINC":
rev = ud.method.latest_revision(ud.url, ud, d, name)
return rev
def getcset(d, depot, host, user, pswd, parm):
p4opt = ""
if "cset" in parm:
return parm["cset"]
if user:
p4opt += " -u %s" % (user)
if pswd:
p4opt += " -P %s" % (pswd)
if host:
p4opt += " -p %s" % (host)
p4date = data.getVar("P4DATE", d, True)
if "revision" in parm:
depot += "#%s" % (parm["revision"])
elif "label" in parm:
depot += "@%s" % (parm["label"])
elif p4date:
depot += "@%s" % (p4date)
p4cmd = data.getVar('FETCHCOMMAND_p4', d, True)
logger.debug(1, "Running %s%s changes -m 1 %s", p4cmd, p4opt, depot)
p4file, errors = bb.process.run("%s%s changes -m 1 %s" %
(p4cmd, p4opt, depot))
cset = p4file.strip()
logger.debug(1, "READ %s", cset)
if not cset:
return -1
return cset.split(' ')[1]
def urldata_init(self, ud, d):
"""
init ClearCase specific variable within url data
"""
ud.proto = "https"
if 'protocol' in ud.parm:
ud.proto = ud.parm['protocol']
if not ud.proto in ('http', 'https'):
raise fetch2.ParameterError("Invalid protocol type", ud.url)
ud.vob = ''
if 'vob' in ud.parm:
ud.vob = ud.parm['vob']
else:
msg = ud.url+": vob must be defined so the fetcher knows what to get."
raise MissingParameterError('vob', msg)
if 'module' in ud.parm:
ud.module = ud.parm['module']
else:
ud.module = ""
ud.basecmd = d.getVar("FETCHCMD_ccrc", True) or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
if data.getVar("SRCREV", d, True) == "INVALID":
raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
ud.label = d.getVar("SRCREV")
ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True)
ud.server = "%s://%s%s" % (ud.proto, ud.host, ud.path)
ud.identifier = "clearcase-%s%s-%s" % ( ud.vob.replace("/", ""),
ud.module.replace("/", "."),
ud.label.replace("/", "."))
ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
ud.csname = "%s-config-spec" % (ud.identifier)
ud.ccasedir = os.path.join(data.getVar("DL_DIR", d, True), ud.type)
ud.viewdir = os.path.join(ud.ccasedir, ud.viewname)
ud.configspecfile = os.path.join(ud.ccasedir, ud.csname)
ud.localfile = "%s.tar.gz" % (ud.identifier)
self.debug("host = %s" % ud.host)
self.debug("path = %s" % ud.path)
self.debug("server = %s" % ud.server)
self.debug("proto = %s" % ud.proto)
self.debug("type = %s" % ud.type)
self.debug("vob = %s" % ud.vob)
self.debug("module = %s" % ud.module)
self.debug("basecmd = %s" % ud.basecmd)
self.debug("label = %s" % ud.label)
self.debug("ccasedir = %s" % ud.ccasedir)
self.debug("viewdir = %s" % ud.viewdir)
self.debug("viewname = %s" % ud.viewname)
self.debug("configspecfile = %s" % ud.configspecfile)
self.debug("localfile = %s" % ud.localfile)
ud.localfile = os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
def exec_func(func, d, dirs=None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
if body is None:
logger.warn("Function %s doesn't exist", func)
return
flags = data.getVarFlags(func, d)
cleandirs = flags.get('cleandirs')
if cleandirs:
for cdir in data.expand(cleandirs, d).split():
bb.utils.remove(cdir, True)
if dirs is None:
dirs = flags.get('dirs')
if dirs:
dirs = data.expand(dirs, d).split()
if dirs:
for adir in dirs:
bb.utils.mkdirhier(adir)
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
bb.utils.mkdirhier(adir)
ispython = flags.get('python')
lockflag = flags.get('lockfiles')
if lockflag:
lockfiles = [data.expand(f, d) for f in lockflag.split()]
else:
lockfiles = None
tempdir = data.getVar('T', d, 1)
# or func allows items to be executed outside of the normal
# task set, such as buildhistory
task = data.getVar('BB_RUNTASK', d, 1) or func
if task == func:
taskfunc = task
else:
taskfunc = "%s.%s" % (task, func)
runfmt = data.getVar('BB_RUNFMT', d, 1) or "run.{func}.{pid}"
runfn = runfmt.format(taskfunc=taskfunc,
task=task,
func=func,
pid=os.getpid())
runfile = os.path.join(tempdir, runfn)
bb.utils.mkdirhier(os.path.dirname(runfile))
with bb.utils.fileslocked(lockfiles):
if ispython:
exec_func_python(func, d, runfile, cwd=adir)
else:
exec_func_shell(func, d, runfile, cwd=adir)
def urldata_init(self, ud, d):
"""
init git specific variable within url data
so that the git method like latest_revision() can work
"""
if 'protocol' in ud.parm:
ud.proto = ud.parm['protocol']
elif not ud.host:
ud.proto = 'file'
else:
ud.proto = "git"
if not ud.proto in ('git', 'file', 'ssh', 'http', 'https', 'rsync'):
raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
ud.nocheckout = ud.parm.get("nocheckout","0") == "1"
ud.rebaseable = ud.parm.get("rebaseable","0") == "1"
# bareclone implies nocheckout
ud.bareclone = ud.parm.get("bareclone","0") == "1"
if ud.bareclone:
ud.nocheckout = 1
branches = ud.parm.get("branch", "master").split(',')
if len(branches) != len(ud.names):
raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
ud.branches = {}
for name in ud.names:
branch = branches[ud.names.index(name)]
ud.branches[name] = branch
ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
ud.setup_revisons(d)
for name in ud.names:
# Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
if not ud.revisions[name] or len(ud.revisions[name]) != 40 or (False in [c in "abcdef0123456789" for c in ud.revisions[name]]):
if ud.revisions[name]:
ud.branches[name] = ud.revisions[name]
ud.revisions[name] = self.latest_revision(ud.url, ud, d, name)
gitsrcname = '%s%s' % (ud.host.replace(':','.'), ud.path.replace('/', '.').replace('*', '.'))
# for rebaseable git repo, it is necessary to keep mirror tar ball
# per revision, so that even the revision disappears from the
# upstream repo in the future, the mirror will remain intact and still
# contains the revision
if ud.rebaseable:
for name in ud.names:
gitsrcname = gitsrcname + '_' + ud.revisions[name]
ud.mirrortarball = 'git2_%s.tar.gz' % (gitsrcname)
ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
gitdir = d.getVar("GITDIR", True) or (d.getVar("DL_DIR", True) + "/git2/")
ud.clonedir = os.path.join(gitdir, gitsrcname)
ud.localfile = ud.clonedir
def urldata_init(self, ud, d):
"""
init ClearCase specific variable within url data
"""
ud.proto = "https"
if 'protocol' in ud.parm:
ud.proto = ud.parm['protocol']
if not ud.proto in ('http', 'https'):
raise fetch2.ParameterError("Invalid protocol type", ud.url)
ud.vob = ''
if 'vob' in ud.parm:
ud.vob = ud.parm['vob']
else:
msg = ud.url+": vob must be defined so the fetcher knows what to get."
raise MissingParameterError('vob', msg)
if 'module' in ud.parm:
ud.module = ud.parm['module']
else:
ud.module = ""
ud.basecmd = d.getVar("FETCHCMD_ccrc", True) or spawn.find_executable("cleartool") or spawn.find_executable("rcleartool")
if data.getVar("SRCREV", d, True) == "INVALID":
raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")
ud.label = d.getVar("SRCREV", False)
ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC", True)
ud.server = "%s://%s%s" % (ud.proto, ud.host, ud.path)
ud.identifier = "clearcase-%s%s-%s" % ( ud.vob.replace("/", ""),
ud.module.replace("/", "."),
ud.label.replace("/", "."))
ud.viewname = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
ud.csname = "%s-config-spec" % (ud.identifier)
ud.ccasedir = os.path.join(data.getVar("DL_DIR", d, True), ud.type)
ud.viewdir = os.path.join(ud.ccasedir, ud.viewname)
ud.configspecfile = os.path.join(ud.ccasedir, ud.csname)
ud.localfile = "%s.tar.gz" % (ud.identifier)
self.debug("host = %s" % ud.host)
self.debug("path = %s" % ud.path)
self.debug("server = %s" % ud.server)
self.debug("proto = %s" % ud.proto)
self.debug("type = %s" % ud.type)
self.debug("vob = %s" % ud.vob)
self.debug("module = %s" % ud.module)
self.debug("basecmd = %s" % ud.basecmd)
self.debug("label = %s" % ud.label)
self.debug("ccasedir = %s" % ud.ccasedir)
self.debug("viewdir = %s" % ud.viewdir)
self.debug("viewname = %s" % ud.viewname)
self.debug("configspecfile = %s" % ud.configspecfile)
self.debug("localfile = %s" % ud.localfile)
ud.localfile = os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
def handle(fn, d, include):
global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__
__body__ = []
__infunc__ = ""
__classname__ = ""
__residue__ = []
if include == 0:
logger.debug(2, "BB %s: handle(data)", fn)
else:
logger.debug(2, "BB %s: handle(data, include)", fn)
base_name = os.path.basename(fn)
(root, ext) = os.path.splitext(base_name)
init(d)
if ext == ".bbclass":
__classname__ = root
classes.append(__classname__)
__inherit_cache = data.getVar('__inherit_cache', d) or []
if not fn in __inherit_cache:
__inherit_cache.append(fn)
data.setVar('__inherit_cache', __inherit_cache, d)
if include != 0:
oldfile = data.getVar('FILE', d)
else:
oldfile = None
abs_fn = resolve_file(fn, d)
if include:
bb.parse.mark_dependency(d, abs_fn)
# actual loading
statements = get_statements(fn, abs_fn, base_name)
# DONE WITH PARSING... time to evaluate
if ext != ".bbclass":
data.setVar('FILE', abs_fn, d)
statements.eval(d)
if ext == ".bbclass":
classes.remove(__classname__)
else:
if include == 0:
return ast.multi_finalize(fn, d)
if oldfile:
d.setVar("FILE", oldfile)
# we have parsed the bb class now
if ext == ".bbclass" or ext == ".inc":
bb.methodpool.get_parsed_dict()[base_name] = 1
return d
def go(self, loc, ud, d):
"""Fetch url"""
if os.access(os.path.join(data.getVar("DL_DIR", d, True), ud.localfile), os.R_OK):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
return
if ud.user:
username = ud.user + '@'
else:
username = ""
gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (ud.tag)
codir = os.path.join(repodir, coname)
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d)
os.chdir(repodir)
# Remove all but the .git directory
if not self._contains_ref(ud.tag, d):
runfetchcmd("rm * -Rf", d)
runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
runfetchcmd("git prune-packed", d)
runfetchcmd("git pack-redundant --all | xargs -r rm", d)
os.chdir(repodir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0":
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
if os.path.exists(codir):
bb.utils.prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
runfetchcmd("git read-tree %s" % (ud.tag), d)
runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
os.chdir(repodir)
bb.utils.prunedir(codir)
def handle(fn, d, include):
global __func_start_regexp__, __inherit_regexp__, __export_func_regexp__, __addtask_regexp__, __addhandler_regexp__, __infunc__, __body__, __residue__
__body__ = []
__infunc__ = ""
__classname__ = ""
__residue__ = []
if include == 0:
logger.debug(2, "BB %s: handle(data)", fn)
else:
logger.debug(2, "BB %s: handle(data, include)", fn)
base_name = os.path.basename(fn)
(root, ext) = os.path.splitext(base_name)
init(d)
if ext == ".bbclass":
__classname__ = root
classes.append(__classname__)
__inherit_cache = data.getVar('__inherit_cache', d) or []
if not fn in __inherit_cache:
__inherit_cache.append(fn)
data.setVar('__inherit_cache', __inherit_cache, d)
if include != 0:
oldfile = data.getVar('FILE', d)
else:
oldfile = None
abs_fn = resolve_file(fn, d)
if include:
bb.parse.mark_dependency(d, abs_fn)
# actual loading
statements = get_statements(fn, abs_fn, base_name)
# DONE WITH PARSING... time to evaluate
if ext != ".bbclass":
data.setVar('FILE', fn, d)
statements.eval(d)
if ext == ".bbclass":
classes.remove(__classname__)
else:
if include == 0:
return ast.multi_finalize(fn, d)
if oldfile:
bb.data.setVar("FILE", oldfile, d)
# we have parsed the bb class now
if ext == ".bbclass" or ext == ".inc":
bb.methodpool.get_parsed_dict()[base_name] = 1
return d
def go(self, loc, ud, d):
"""Fetch url"""
if Fetch.try_mirror(d, ud.localfile):
bb.msg.debug(1, bb.msg.domain.Fetcher, "%s already exists (or was stashed). Skipping git checkout." % ud.localpath)
return
if ud.user:
username = ud.user + '@'
else:
username = ""
gitsrcname = '%s%s' % (ud.host, ud.path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (ud.tag)
codir = os.path.join(repodir, coname)
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
runfetchcmd("tar -xzf %s" % (repofile), d)
else:
runfetchcmd("git clone -n %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, repodir), d)
os.chdir(repodir)
# Remove all but the .git directory
if not self._contains_ref(ud.tag, d):
runfetchcmd("rm * -Rf", d)
runfetchcmd("git fetch %s://%s%s%s %s" % (ud.proto, username, ud.host, ud.path, ud.branch), d)
runfetchcmd("git fetch --tags %s://%s%s%s" % (ud.proto, username, ud.host, ud.path), d)
runfetchcmd("git prune-packed", d)
runfetchcmd("git pack-redundant --all | xargs -r rm", d)
os.chdir(repodir)
mirror_tarballs = data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True)
if mirror_tarballs != "0":
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git repository")
runfetchcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ), d)
if os.path.exists(codir):
bb.utils.prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
runfetchcmd("git read-tree %s" % (ud.tag), d)
runfetchcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")), d)
os.chdir(codir)
bb.msg.note(1, bb.msg.domain.Fetcher, "Creating tarball of git checkout")
runfetchcmd("tar -czf %s %s" % (ud.localpath, os.path.join(".", "*") ), d)
os.chdir(repodir)
bb.utils.prunedir(codir)
def urldata_init(self, ud, d):
"""
init git specific variable within url data
so that the git method like latest_revision() can work
"""
if "protocol" in ud.parm:
ud.proto = ud.parm["protocol"]
elif not ud.host:
ud.proto = "file"
else:
ud.proto = "git"
if not ud.proto in ("git", "file", "ssh", "http", "https", "rsync"):
raise bb.fetch2.ParameterError("Invalid protocol type", ud.url)
ud.nocheckout = ud.parm.get("nocheckout", "0") == "1"
ud.rebaseable = ud.parm.get("rebaseable", "0") == "1"
branches = ud.parm.get("branch", "master").split(",")
if len(branches) != len(ud.names):
raise bb.fetch2.ParameterError("The number of name and branch parameters is not balanced", ud.url)
ud.branches = {}
for name in ud.names:
branch = branches[ud.names.index(name)]
ud.branches[name] = branch
ud.basecmd = data.getVar("FETCHCMD_git", d, True) or "git"
ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0") or ud.rebaseable
ud.setup_revisons(d)
for name in ud.names:
# Ensure anything that doesn't look like a sha256 checksum/revision is translated into one
if (
not ud.revisions[name]
or len(ud.revisions[name]) != 40
or (False in [c in "abcdef0123456789" for c in ud.revisions[name]])
):
ud.branches[name] = ud.revisions[name]
ud.revisions[name] = self.latest_revision(ud.url, ud, d, name)
gitsrcname = "%s%s" % (ud.host, ud.path.replace("/", "."))
# for rebaseable git repo, it is necessary to keep mirror tar ball
# per revision, so that even the revision disappears from the
# upstream repo in the future, the mirror will remain intact and still
# contains the revision
if ud.rebaseable:
for name in ud.names:
gitsrcname = gitsrcname + "_" + ud.revisions[name]
ud.mirrortarball = "git2_%s.tar.gz" % (gitsrcname)
ud.fullmirror = os.path.join(data.getVar("DL_DIR", d, True), ud.mirrortarball)
ud.clonedir = os.path.join(data.expand("${GITDIR}", d), gitsrcname)
ud.localfile = ud.clonedir
def testGVar(self):
# import the data module
from bb import data
from bb import data_smart
d = data_smart.DataSmart()
data.setVar('TEST', 'testcontents', d )
self.assertEquals( data.getVar('TEST',d), 'testcontents', 'Setting Variable Failed')
data.delVar('TEST', d)
self.assertEquals(data.getVar('TEST', d), None)
def test_assign():
print "Testing assign"
data = bb.data.init()
a = ast.Assignment('KEY', 'VALUE')
a.eval(data, None)
print "\t", data.getVar('KEY', False) == 'VALUE'
a = ast.Assignment('KEY', 'NEWVALUE')
a.eval(data, None)
print "\t", data.getVar('KEY', False) == 'NEWVALUE'
def test_immediate():
print "Testing immediate assignment"
data = bb.data.init()
a = ast.ImmediateAssignment('KEY', 'VALUE')
a.eval(data, None)
print "\t", data.getVar('KEY', False) == 'VALUE'
a = ast.ImmediateAssignment('KEY', '${@3*3}')
a.eval(data, None)
print "\t", data.getVar('KEY', False) == '9'
def exec_func(func, d, dirs=None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
flags = data.getVarFlags(func, d)
for item in [
'deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs',
'lockfiles', 'fakeroot'
]:
if not item in flags:
flags[item] = None
ispython = flags['python']
cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if dirs:
dirs = data.expand(dirs, d)
else:
dirs = (data.expand(flags['dirs'], d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.getVar('TOPDIR', d, True)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
locks = []
lockfiles = (data.expand(flags['lockfiles'], d) or "").split()
for lock in lockfiles:
locks.append(bb.utils.lockfile(lock))
if flags['python']:
exec_func_python(func, d)
else:
exec_func_shell(func, d, flags)
for lock in locks:
bb.utils.unlockfile(lock)
if os.path.exists(prevdir):
os.chdir(prevdir)
def extract_stamp_data(d, fn):
"""
Extracts stamp data from d which is either a data dictonary (fn unset)
or a dataCache entry (fn set).
"""
if fn:
return (d.task_queues[fn], d.stamp[fn], d.task_deps[fn])
task_graph = data.getVar('_task_graph', d)
if not task_graph:
task_graph = bb.digraph()
data.setVar('_task_graph', task_graph, d)
return (task_graph, data.getVar('STAMP', d, 1), None)
def parse( self, params ):
"""(Re-)parse .bb files and calculate the dependency graph"""
cooker.status = cache.CacheData()
ignore = data.getVar("ASSUME_PROVIDED", cooker.configuration.data, 1) or ""
cooker.status.ignored_dependencies = set( ignore.split() )
cooker.handleCollections( data.getVar("BBFILE_COLLECTIONS", cooker.configuration.data, 1) )
(filelist, masked) = cooker.collect_bbfiles()
cooker.parse_bbfiles(filelist, masked, cooker.myProgressCallback)
cooker.buildDepgraph()
global parsed
parsed = True
print
def exec_func(func, d, dirs=None, logfile=NULL):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
if body is None:
logger.warn("Function %s doesn't exist", func)
return
flags = data.getVarFlags(func, d)
cleandirs = flags.get('cleandirs')
if cleandirs:
for cdir in data.expand(cleandirs, d).split():
bb.utils.remove(cdir, True)
if dirs is None:
dirs = flags.get('dirs')
if dirs:
dirs = data.expand(dirs, d).split()
if dirs:
for adir in dirs:
bb.utils.mkdirhier(adir)
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
if not os.path.exists(adir):
adir = None
ispython = flags.get('python')
fakeroot = flags.get('fakeroot')
lockflag = flags.get('lockfiles')
if lockflag:
lockfiles = [data.expand(f, d) for f in lockflag.split()]
else:
lockfiles = None
tempdir = data.getVar('T', d, 1)
runfile = os.path.join(tempdir, 'run.{0}.{1}'.format(func, os.getpid()))
with bb.utils.fileslocked(lockfiles):
if ispython:
exec_func_python(func, d, runfile, logfile, cwd=adir)
else:
exec_func_shell(func,
d,
runfile,
logfile,
cwd=adir,
fakeroot=fakeroot)
def inherit(files, fn, lineno, d):
__inherit_cache = data.getVar('__inherit_cache', d) or []
files = d.expand(files).split()
for file in files:
if not os.path.isabs(file) and not file.endswith(".bbclass"):
file = os.path.join('classes', '%s.bbclass' % file)
if not file in __inherit_cache:
logger.log(logging.DEBUG - 1, "BB %s:%d: inheriting %s", fn,
lineno, file)
__inherit_cache.append(file)
data.setVar('__inherit_cache', __inherit_cache, d)
include(fn, file, lineno, d, "inherit")
__inherit_cache = data.getVar('__inherit_cache', d) or []
def __init__(self, data):
self.basehash = {}
self.taskhash = {}
self.taskdeps = {}
self.runtaskdeps = {}
self.gendeps = {}
self.lookupcache = {}
self.basewhitelist = set((data.getVar("BB_HASHBASE_WHITELIST", True) or "").split())
self.taskwhitelist = data.getVar("BB_HASHTASK_WHITELIST", True) or None
if self.taskwhitelist:
self.twl = re.compile(self.taskwhitelist)
else:
self.twl = None
def localcount_internal_helper(ud, d):
"""
Return:
a) a locked localcount if specified
b) None otherwise
"""
localcount = None
if 'name' in ud.parm:
pn = data.getVar("PN", d, 1)
localcount = data.getVar("LOCALCOUNT_" + ud.parm['name'], d, 1)
if not localcount:
localcount = data.getVar("LOCALCOUNT", d, 1)
return localcount
def inherit(files, d):
__inherit_cache = data.getVar('__inherit_cache', d) or []
fn = ""
lineno = 0
files = data.expand(files, d)
for file in files:
if file[0] != "/" and file[-8:] != ".bbclass":
file = os.path.join('classes', '%s.bbclass' % file)
if not file in __inherit_cache:
bb.msg.debug(2, bb.msg.domain.Parsing, "BB %s:%d: inheriting %s" % (fn, lineno, file))
__inherit_cache.append( file )
data.setVar('__inherit_cache', __inherit_cache, d)
include(fn, file, d, "inherit")
__inherit_cache = data.getVar('__inherit_cache', d) or []
def doparse(url, d):
parm = {}
path = url.split("://")[1]
delim = path.find("@");
if delim != -1:
(user, pswd, host, port) = path.split('@')[0].split(":")
path = path.split('@')[1]
else:
(host, port) = data.getVar('P4PORT', d).split(':')
user = ""
pswd = ""
if path.find(";") != -1:
keys=[]
values=[]
plist = path.split(';')
for item in plist:
if item.count('='):
(key, value) = item.split('=')
keys.append(key)
values.append(value)
parm = dict(zip(keys, values))
path = "//" + path.split(';')[0]
host += ":%s" % (port)
parm["cset"] = Perforce.getcset(d, path, host, user, pswd, parm)
return host, path, user, pswd, parm
def init_rundepcheck(self, data):
self.taskhash_ignore_tasks = data.getVar(
"BB_TASKHASH_IGNORE_TASKS") or None
if self.taskhash_ignore_tasks:
self.twl = re.compile(self.taskhash_ignore_tasks)
else:
self.twl = None
def parseConfigurationFile(self, afile):
try:
self.configuration.data = bb.parse.handle(afile,
self.configuration.data)
# Handle any INHERITs and inherit the base class
inherits = ["base"] + (bb.data.getVar(
'INHERIT', self.configuration.data, True) or "").split()
for inherit in inherits:
self.configuration.data = bb.parse.handle(
os.path.join('classes', '%s.bbclass' % inherit),
self.configuration.data, True)
# Nomally we only register event handlers at the end of parsing .bb files
# We register any handlers we've found so far here...
for var in data.getVar('__BBHANDLERS',
self.configuration.data) or []:
bb.event.register(var,
bb.data.getVar(var, self.configuration.data))
bb.fetch.fetcher_init(self.configuration.data)
bb.event.fire(bb.event.ConfigParsed(self.configuration.data))
except IOError, e:
bb.msg.fatal(bb.msg.domain.Parsing,
"Error when parsing %s: %s" % (afile, str(e)))
def urldata_init(self, ud, d):
"""
init NPM specific variable within url data
"""
if 'downloadfilename' in ud.parm:
ud.basename = ud.parm['downloadfilename']
else:
ud.basename = os.path.basename(ud.path)
# can't call it ud.name otherwise fetcher base class will start doing sha1stuff
# TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
ud.pkgname = ud.parm.get("name", None)
if not ud.pkgname:
raise ParameterError("NPM fetcher requires a name parameter", ud.url)
ud.version = ud.parm.get("version", None)
if not ud.version:
raise ParameterError("NPM fetcher requires a version parameter", ud.url)
ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
prefixdir = "npm/%s" % ud.pkgname
ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
if not os.path.exists(ud.pkgdatadir):
bb.utils.mkdirhier(ud.pkgdatadir)
ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)
self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
ud.prefixdir = prefixdir
ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0")
ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
def localpath(self, url, ud, d):
if not "module" in ud.parm:
raise MissingParameterError("hg method needs a 'module' parameter")
ud.module = ud.parm["module"]
# Create paths to mercurial checkouts
relpath = self._strip_leading_slashes(ud.path)
ud.pkgdir = os.path.join(data.expand('${HGDIR}', d), ud.host, relpath)
ud.moddir = os.path.join(ud.pkgdir, ud.module)
if 'rev' in ud.parm:
ud.revision = ud.parm['rev']
else:
tag = Fetch.srcrev_internal_helper(ud, d)
if tag is True:
ud.revision = self.latest_revision(url, ud, d)
elif tag:
ud.revision = tag
else:
ud.revision = self.latest_revision(url, ud, d)
ud.localfile = data.expand('%s_%s_%s_%s.tar.gz' % (ud.module.replace('/', '.'), ud.host, ud.path.replace('/', '.'), ud.revision), d)
return os.path.join(data.getVar("DL_DIR", d, True), ud.localfile)
def which(self, params):
"""Computes the providers for a given providee"""
# Need to use taskData for this information
item = params[0]
self._checkParsed()
preferred = data.getVar("PREFERRED_PROVIDER_%s" % item,
cooker.configuration.data, 1)
if not preferred: preferred = item
try:
lv, lf, pv, pf = Providers.findBestProvider(
preferred, cooker.configuration.data, cooker.status)
except KeyError:
lv, lf, pv, pf = (None, ) * 4
try:
providers = cooker.status.providers[item]
except KeyError:
print("SHELL: ERROR: Nothing provides", preferred)
else:
for provider in providers:
if provider == pf: provider = " (***) %s" % provider
else: provider = " %s" % provider
print(provider)
def localpath(self, url, urldata, d):
"""
Return the local filename of a given url assuming a successful fetch.
Can also setup variables in urldata for use in go (saving code duplication
and duplicate code execution)
"""
return os.path.join(data.getVar("DL_DIR", d, True), urldata.localfile)
def download(self, ud, d):
"""Fetch urls"""
urlo = URI(ud.url)
basecmd = 'sftp -oBatchMode=yes'
port = ''
if urlo.port:
port = '-P %d' % urlo.port
urlo.port = None
dldir = data.getVar('DL_DIR', d, True)
lpath = os.path.join(dldir, ud.localfile)
user = ''
if urlo.userinfo:
user = urlo.userinfo + '@'
path = urlo.path
# Supoprt URIs relative to the user's home directory, with
# the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>).
if path[:3] == '/~/':
path = path[3:]
remote = '%s%s:%s' % (user, urlo.hostname, path)
cmd = '%s %s %s %s' % (basecmd, port, remote, lpath)
bb.fetch2.check_network_access(d, cmd, ud.url)
runfetchcmd(cmd, d)
return True
def load_bbfile( self, bbfile , config):
"""
Load and parse one .bb build file
Return the data and whether parsing resulted in the file being skipped
"""
import bb
from bb import utils, data, parse, debug, event, fatal
# expand tmpdir to include this topdir
data.setVar('TMPDIR', data.getVar('TMPDIR', config, 1) or "", config)
bbfile_loc = os.path.abspath(os.path.dirname(bbfile))
oldpath = os.path.abspath(os.getcwd())
if bb.parse.cached_mtime_noerror(bbfile_loc):
os.chdir(bbfile_loc)
bb_data = data.init_db(config)
try:
bb_data = parse.handle(bbfile, bb_data) # read .bb data
os.chdir(oldpath)
return bb_data, False
except bb.parse.SkipPackage:
os.chdir(oldpath)
return bb_data, True
except:
os.chdir(oldpath)
raise
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
return
flags = data.getVarFlags(func, d)
for item in ['deps', 'check', 'interactive', 'python', 'cleandirs', 'dirs', 'lockfiles', 'fakeroot']:
if not item in flags:
flags[item] = None
ispython = flags['python']
cleandirs = (data.expand(flags['cleandirs'], d) or "").split()
for cdir in cleandirs:
os.system("rm -rf %s" % cdir)
if dirs:
dirs = data.expand(dirs, d)
else:
dirs = (data.expand(flags['dirs'], d) or "").split()
for adir in dirs:
mkdirhier(adir)
if len(dirs) > 0:
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
# Save current directory
try:
prevdir = os.getcwd()
except OSError:
prevdir = data.getVar('TOPDIR', d, True)
# Setup logfiles
t = data.getVar('T', d, 1)
if not t:
bb.msg.fatal(bb.msg.domain.Build, "T not set")
mkdirhier(t)
# Gross hack, FIXME
import random
logfile = "%s/log.%s.%s.%s" % (t, func, str(os.getpid()),random.random())
runfile = "%s/run.%s.%s" % (t, func, str(os.getpid()))
# Change to correct directory (if specified)
if adir and os.access(adir, os.F_OK):
os.chdir(adir)
# Handle logfiles
si = file('/dev/null', 'r')
try:
if bb.msg.debug_level['default'] > 0 or ispython:
so = os.popen("tee \"%s\"" % logfile, "w")
else:
so = file(logfile, 'w')
except OSError, e:
bb.msg.error(bb.msg.domain.Build, "opening log file: %s" % e)
pass
def exec_func(func, d, dirs = None):
"""Execute an BB 'function'"""
body = data.getVar(func, d)
if not body:
if body is None:
logger.warn("Function %s doesn't exist", func)
return
flags = data.getVarFlags(func, d)
cleandirs = flags.get('cleandirs')
if cleandirs:
for cdir in data.expand(cleandirs, d).split():
bb.utils.remove(cdir, True)
if dirs is None:
dirs = flags.get('dirs')
if dirs:
dirs = data.expand(dirs, d).split()
if dirs:
for adir in dirs:
bb.utils.mkdirhier(adir)
adir = dirs[-1]
else:
adir = data.getVar('B', d, 1)
bb.utils.mkdirhier(adir)
ispython = flags.get('python')
lockflag = flags.get('lockfiles')
if lockflag:
lockfiles = [data.expand(f, d) for f in lockflag.split()]
else:
lockfiles = None
tempdir = data.getVar('T', d, 1)
# or func allows items to be executed outside of the normal
# task set, such as buildhistory
task = data.getVar('BB_RUNTASK', d, 1) or func
if task == func:
taskfunc = task
else:
taskfunc = "%s.%s" % (task, func)
runfmt = data.getVar('BB_RUNFMT', d, 1) or "run.{func}.{pid}"
runfn = runfmt.format(taskfunc=taskfunc, task=task, func=func, pid=os.getpid())
runfile = os.path.join(tempdir, runfn)
bb.utils.mkdirhier(os.path.dirname(runfile))
with bb.utils.fileslocked(lockfiles):
if ispython:
exec_func_python(func, d, runfile, cwd=adir)
else:
exec_func_shell(func, d, runfile, cwd=adir)
