def get_source_date_epoch_from_git(d, sourcedir):
if not "git://" in d.getVar('SRC_URI'):
return None
gitpath = find_git_folder(d, sourcedir)
if not gitpath:
return None
# Check that the repository has a valid HEAD; it may not if subdir is used
# in SRC_URI
p = subprocess.run(['git', 'rev-parse', 'HEAD'],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
cwd=gitpath)
if p.returncode != 0:
bb.debug(
1, "%s does not have a valid HEAD: %s" %
(gitpath, p.stdout.decode('utf-8')))
return None
bb.debug(1, "git repository: %s" % gitpath)
p = subprocess.run(['git', 'log', '-1', '--pretty=%ct'],
check=True,
stdout=subprocess.PIPE,
cwd=gitpath)
return int(p.stdout.decode('utf-8'))
def download(self, Bucket, Key, Filename, quiet=True):
if self.s3client is None:
self.makeclient()
for attempt in range(10):
try:
info = self.s3client.head_object(Bucket=Bucket, Key=Key)
self.s3client.download_file(Bucket=Bucket,
Key=Key,
Filename=Filename)
if 'LastModified' in info:
mtime = int(time.mktime(info['LastModified'].timetuple()))
os.utime(Filename, (mtime, mtime))
except botocore.exceptions.NoCredentialsError:
s3retry_wait(attempt)
continue
except botocore.exceptions.ClientError as e:
err = e.response['Error']
if quiet and err['Code'] == "404":
bb.debug(2, "not found: {}/{}".format(Bucket, Key))
else:
bb.warn("{}/{}: {} {}".format(Bucket, Key, err['Code'],
err['Message']))
return False
except OSError as e:
if quiet:
pass
bb.warn("os.utime({}): {} (errno {})".format(
Filename, e.strerror, e.errno))
return False
return True
bb.warn("{}/{}: credentials error on download for 10 attempts".format(
Bucket, Key))
return False
def execute(graph, item):
if data.getVarFlag(item, 'task', d):
if item in task_cache:
return 1
if task != item:
# deeper than toplevel, exec w/ deps
exec_task(item, d)
return 1
try:
debug(1, "Executing task %s" % item)
old_overrides = data.getVar('OVERRIDES', d, 0)
localdata = data.createCopy(d)
data.setVar('OVERRIDES', 'task_%s:%s' % (item, old_overrides), localdata)
data.update_data(localdata)
event.fire(TaskStarted(item, localdata))
exec_func(item, localdata)
event.fire(TaskSucceeded(item, localdata))
task_cache.append(item)
except FuncFailed, reason:
note( "Task failed: %s" % reason )
failedevent = TaskFailed(item, d)
event.fire(failedevent)
raise EventException(None, failedevent)
def go(self, d, urls = []):
"""Fetch urls"""
if not urls:
urls = self.urls
for loc in urls:
(type, host, path, user, pswd, parm) = bb.decodeurl(data.expand(loc, d))
tag = gettag(parm)
proto = getprotocol(parm)
gitsrcname = '%s%s' % (host, path.replace('/', '.'))
repofilename = 'git_%s.tar.gz' % (gitsrcname)
repofile = os.path.join(data.getVar("DL_DIR", d, 1), repofilename)
repodir = os.path.join(data.expand('${GITDIR}', d), gitsrcname)
coname = '%s' % (tag)
codir = os.path.join(repodir, coname)
cofile = self.localpath(loc, d)
# tag=="master" must always update
if (tag != "master") and Fetch.try_mirror(d, localfile(loc, d)):
bb.debug(1, "%s already exists (or was stashed). Skipping git checkout." % cofile)
continue
if not os.path.exists(repodir):
if Fetch.try_mirror(d, repofilename):
bb.mkdirhier(repodir)
os.chdir(repodir)
rungitcmd("tar -xzf %s" % (repofile),d)
else:
rungitcmd("git clone -n %s://%s%s %s" % (proto, host, path, repodir),d)
os.chdir(repodir)
rungitcmd("git pull %s://%s%s" % (proto, host, path),d)
rungitcmd("git pull --tags %s://%s%s" % (proto, host, path),d)
rungitcmd("git prune-packed", d)
rungitcmd("git pack-redundant --all | xargs -r rm", d)
# Remove all but the .git directory
rungitcmd("rm * -Rf", d)
# old method of downloading tags
#rungitcmd("rsync -a --verbose --stats --progress rsync://%s%s/ %s" % (host, path, os.path.join(repodir, ".git", "")),d)
os.chdir(repodir)
bb.note("Creating tarball of git repository")
rungitcmd("tar -czf %s %s" % (repofile, os.path.join(".", ".git", "*") ),d)
if os.path.exists(codir):
prunedir(codir)
bb.mkdirhier(codir)
os.chdir(repodir)
rungitcmd("git read-tree %s" % (tag),d)
rungitcmd("git checkout-index -q -f --prefix=%s -a" % (os.path.join(codir, "git", "")),d)
os.chdir(codir)
bb.note("Creating tarball of git checkout")
rungitcmd("tar -czf %s %s" % (cofile, os.path.join(".", "*") ),d)
def exec_func_python(func, d, runfile, cwd=None):
"""Execute a python BB 'function'"""
bbfile = d.getVar('FILE', True)
code = _functionfmt.format(function=func, body=d.getVar(func, True))
bb.utils.mkdirhier(os.path.dirname(runfile))
with open(runfile, 'w') as script:
script.write(code)
if cwd:
try:
olddir = os.getcwd()
except OSError:
olddir = None
os.chdir(cwd)
bb.debug(2, "Executing python function %s" % func)
try:
comp = utils.better_compile(code, func, bbfile)
utils.better_exec(comp, {"d": d}, code, bbfile)
except (bb.parse.SkipRecipe, bb.build.FuncFailed):
raise
except:
raise FuncFailed(func, None)
finally:
bb.debug(2, "Python function %s finished" % func)
if cwd and olddir:
try:
os.chdir(olddir)
except OSError:
pass
def exec_func_python(func, d, runfile, cwd=None):
"""Execute a python BB 'function'"""
bbfile = d.getVar('FILE', True)
code = _functionfmt.format(function=func, body=d.getVar(func, True))
bb.utils.mkdirhier(os.path.dirname(runfile))
with open(runfile, 'w') as script:
script.write(code)
if cwd:
try:
olddir = os.getcwd()
except OSError:
olddir = None
os.chdir(cwd)
bb.debug(2, "Executing python function %s" % func)
try:
comp = utils.better_compile(code, func, bbfile)
utils.better_exec(comp, {"d": d}, code, bbfile)
except:
if sys.exc_info()[0] in (bb.parse.SkipPackage, bb.build.FuncFailed):
raise
raise FuncFailed(func, None)
finally:
bb.debug(2, "Python function %s finished" % func)
if cwd and olddir:
try:
os.chdir(olddir)
except OSError:
pass
def fetch_uri(uri, basename, dl, md5, d):
if os.path.exists(dl):
# file exists, but we didnt complete it.. trying again..
fetchcmd = bb.data.getVar("RESUMECOMMAND", d, 1)
else:
fetchcmd = bb.data.getVar("FETCHCOMMAND", d, 1)
bb.note("fetch " + uri)
fetchcmd = fetchcmd.replace("${URI}", uri)
fetchcmd = fetchcmd.replace("${FILE}", basename)
bb.debug(2, "executing " + fetchcmd)
ret = os.system(fetchcmd)
if ret != 0:
return False
# supposedly complete.. write out md5sum
if bb.which(bb.data.getVar("PATH", d), "md5sum"):
try:
md5pipe = os.popen("md5sum " + dl)
md5data = (md5pipe.readline().split() or [""])[0]
md5pipe.close()
except OSError:
md5data = ""
md5out = file(md5, "w")
md5out.write(md5data)
md5out.close()
else:
md5out = file(md5, "w")
md5out.write("")
md5out.close()
return True
def readfifo(data):
lines = data.split('\0')
for line in lines:
splitval = line.split(' ', 1)
cmd = splitval[0]
if len(splitval) > 1:
value = splitval[1]
else:
value = ''
if cmd == 'bbplain':
bb.plain(value)
elif cmd == 'bbnote':
bb.note(value)
elif cmd == 'bbwarn':
bb.warn(value)
elif cmd == 'bberror':
bb.error(value)
elif cmd == 'bbfatal':
# The caller will call exit themselves, so bb.error() is
# what we want here rather than bb.fatal()
bb.error(value)
elif cmd == 'bbfatal_log':
bb.error(value, forcelog=True)
elif cmd == 'bbdebug':
splitval = value.split(' ', 1)
level = int(splitval[0])
value = splitval[1]
bb.debug(level, value)
def run(d, cmd, *args):
import subprocess
topdir = d.getVar('TMPDIR', True)
toolchain_path = d.getVar('EXTERNAL_TOOLCHAIN', True)
if toolchain_path:
target_prefix = d.getVar('EXTERNAL_TARGET_SYS', True) + '-'
path = os.path.join(toolchain_path, 'bin', target_prefix + cmd)
args = [path] + list(args)
try:
output = oe.path.check_output(args,
cwd=topdir,
stderr=subprocess.STDOUT)
except oe.path.CalledProcessError as exc:
import pipes
bb.debug(
1, "{0} failed: {1}".format(
' '.join(pipes.quote(a) for a in args), exc.output))
except OSError as exc:
import pipes
bb.debug(
1, "{0} failed: {1}".format(
' '.join(pipes.quote(a) for a in args), str(exc)))
else:
return output
return 'UNKNOWN'
def try_mirror(d, tarfn):
"""
Try to use a mirrored version of the sources. We do this
to avoid massive loads on foreign cvs and svn servers.
This method will be used by the different fetcher
implementations.
d Is a bb.data instance
tarfn is the name of the tarball
"""
tarpath = os.path.join(data.getVar("DL_DIR", d, 1), tarfn)
if os.access(tarpath, os.R_OK):
bb.debug(1, "%s already exists, skipping checkout." % tarfn)
return True
pn = data.getVar('PN', d, True)
src_tarball_stash = None
if pn:
src_tarball_stash = (data.getVar('SRC_TARBALL_STASH_%s' % pn, d, True) or data.getVar('CVS_TARBALL_STASH_%s' % pn, d, True) or data.getVar('SRC_TARBALL_STASH', d, True) or data.getVar('CVS_TARBALL_STASH', d, True) or "").split()
for stash in src_tarball_stash:
fetchcmd = data.getVar("FETCHCOMMAND_mirror", d, True) or data.getVar("FETCHCOMMAND_wget", d, True)
uri = stash + tarfn
bb.note("fetch " + uri)
fetchcmd = fetchcmd.replace("${URI}", uri)
ret = os.system(fetchcmd)
if ret == 0:
bb.note("Fetched %s from tarball stash, skipping checkout" % tarfn)
return True
return False
def readfifo(data):
lines = data.split(b"\0")
for line in lines:
splitval = line.split(b" ", 1)
cmd = splitval[0]
if len(splitval) > 1:
value = splitval[1].decode("utf-8")
else:
value = ""
if cmd == "bbplain":
bb.plain(value)
elif cmd == "bbnote":
bb.note(value)
elif cmd == "bbwarn":
bb.warn(value)
elif cmd == "bberror":
bb.error(value)
elif cmd == "bbfatal":
# The caller will call exit themselves, so bb.error() is
# what we want here rather than bb.fatal()
bb.error(value)
elif cmd == "bbfatal_log":
bb.error(value, forcelog=True)
elif cmd == "bbdebug":
splitval = value.split(" ", 1)
level = int(splitval[0])
value = splitval[1]
bb.debug(level, value)
def uri_replace(uri, uri_find, uri_replace, d=bb.data.init()):
# bb.note("uri_replace: operating on %s" % uri)
if not uri or not uri_find or not uri_replace:
bb.debug(1, "uri_replace: passed an undefined value, not replacing")
uri_decoded = list(bb.decodeurl(uri))
uri_find_decoded = list(bb.decodeurl(uri_find))
uri_replace_decoded = list(bb.decodeurl(uri_replace))
result_decoded = ["", "", "", "", "", {}]
for i in uri_find_decoded:
loc = uri_find_decoded.index(i)
result_decoded[loc] = uri_decoded[loc]
import types
if type(i) == types.StringType:
import re
if re.match(i, uri_decoded[loc]):
result_decoded[loc] = re.sub(i, uri_replace_decoded[loc], uri_decoded[loc])
if uri_find_decoded.index(i) == 2:
if d:
localfn = bb.fetch.localpath(uri, d)
if localfn:
result_decoded[loc] = (
os.path.dirname(result_decoded[loc])
+ "/"
+ os.path.basename(bb.fetch.localpath(uri, d))
)
# bb.note("uri_replace: matching %s against %s and replacing with %s" % (i, uri_decoded[loc], uri_replace_decoded[loc]))
else:
# bb.note("uri_replace: no match")
return uri
# else:
# for j in i.keys():
# FIXME: apply replacements against options
return bb.encodeurl(result_decoded)
def _initialize_intercepts(self):
bb.note("Initializing intercept dir for %s" % self.target_rootfs)
# As there might be more than one instance of PackageManager operating at the same time
# we need to isolate the intercept_scripts directories from each other,
# hence the ugly hash digest in dir name.
self.intercepts_dir = os.path.join(
self.d.getVar('WORKDIR'), "intercept_scripts-%s" %
(hashlib.sha256(self.target_rootfs.encode()).hexdigest()))
postinst_intercepts = (self.d.getVar("POSTINST_INTERCEPTS")
or "").split()
if not postinst_intercepts:
postinst_intercepts_path = self.d.getVar(
"POSTINST_INTERCEPTS_PATH")
if not postinst_intercepts_path:
postinst_intercepts_path = self.d.getVar(
"POSTINST_INTERCEPTS_DIR") or self.d.expand(
"${COREBASE}/scripts/postinst-intercepts")
postinst_intercepts = oe.path.which_wild('*',
postinst_intercepts_path)
bb.debug(
1, 'Collected intercepts:\n%s' %
''.join(' %s\n' % i for i in postinst_intercepts))
bb.utils.remove(self.intercepts_dir, True)
bb.utils.mkdirhier(self.intercepts_dir)
for intercept in postinst_intercepts:
bb.utils.copyfile(
intercept,
os.path.join(self.intercepts_dir, os.path.basename(intercept)))
def run_ostree(self, command, *args, **kwargs):
cmd = 'ostree ' + self.format(command, *args, **kwargs)
bb.debug(1, 'Running: {0}'.format(cmd))
output = subprocess.check_output(cmd,
shell=True,
stderr=subprocess.STDOUT)
return output
def prserv_import_db(d, filter_version=None, filter_pkgarch=None, filter_checksum=None):
if d.getVar('USE_PR_SERV', True) != "1":
bb.error("Not using network based PR service")
return None
conn = d.getVar("__PRSERV_CONN", True)
if conn is None:
conn = prserv_make_conn(d)
if conn is None:
bb.error("Making connection failed to remote PR service")
return None
#get the entry values
imported = []
prefix = "PRAUTO$"
for v in d.keys():
if v.startswith(prefix):
(remain, sep, checksum) = v.rpartition('$')
(remain, sep, pkgarch) = remain.rpartition('$')
(remain, sep, version) = remain.rpartition('$')
if (remain + '$' != prefix) or \
(filter_version and filter_version != version) or \
(filter_pkgarch and filter_pkgarch != pkgarch) or \
(filter_checksum and filter_checksum != checksum):
continue
try:
value = int(d.getVar(remain + '$' + version + '$' + pkgarch + '$' + checksum, True))
except BaseException as exc:
bb.debug("Not valid value of %s:%s" % (v,str(exc)))
continue
ret = conn.importone(version,pkgarch,checksum,value)
if ret != value:
bb.error("importing(%s,%s,%s,%d) failed. DB may have larger value %d" % (version,pkgarch,checksum,value,ret))
else:
imported.append((version,pkgarch,checksum,value))
return imported
def exec_func_python(func, d, runfile, cwd=None):
"""Execute a python BB 'function'"""
code = _functionfmt.format(function=func)
bb.utils.mkdirhier(os.path.dirname(runfile))
with open(runfile, 'w') as script:
bb.data.emit_func_python(func, script, d)
if cwd:
try:
olddir = os.getcwd()
except OSError as e:
bb.warn("%s: Cannot get cwd: %s" % (func, e))
olddir = None
os.chdir(cwd)
bb.debug(2, "Executing python function %s" % func)
try:
text = "def %s(d):\n%s" % (func, d.getVar(func, False))
fn = d.getVarFlag(func, "filename", False)
lineno = int(d.getVarFlag(func, "lineno", False))
bb.methodpool.insert_method(func, text, fn, lineno - 1)
comp = utils.better_compile(code, func, "exec_python_func() autogenerated")
utils.better_exec(comp, {"d": d}, code, "exec_python_func() autogenerated")
finally:
bb.debug(2, "Python function %s finished" % func)
if cwd and olddir:
try:
os.chdir(olddir)
except OSError as e:
bb.warn("%s: Cannot restore cwd %s: %s" % (func, olddir, e))
def readfifo(data):
lines = data.split('\0')
for line in lines:
splitval = line.split(' ', 1)
cmd = splitval[0]
if len(splitval) > 1:
value = splitval[1]
else:
value = ''
if cmd == 'bbplain':
bb.plain(value)
elif cmd == 'bbnote':
bb.note(value)
elif cmd == 'bbwarn':
bb.warn(value)
elif cmd == 'bberror':
bb.error(value)
elif cmd == 'bbfatal':
# The caller will call exit themselves, so bb.error() is
# what we want here rather than bb.fatal()
bb.error(value)
elif cmd == 'bbfatal_log':
bb.error(value, forcelog=True)
elif cmd == 'bbdebug':
splitval = value.split(' ', 1)
level = int(splitval[0])
value = splitval[1]
bb.debug(level, value)
def fixed_source_date_epoch(d):
bb.debug(1, "No tarball or git repo found to determine SOURCE_DATE_EPOCH")
source_date_epoch = d.getVar('SOURCE_DATE_EPOCH_FALLBACK')
if source_date_epoch:
bb.debug(1, "Using SOURCE_DATE_EPOCH_FALLBACK")
return int(source_date_epoch)
return 0
def get_controller_modulenames(self, bbpath):
controllerslist = []
def add_controller_list(path):
if not os.path.exists(os.path.join(path, '__init__.py')):
bb.fatal(
'Controllers directory %s exists but is missing __init__.py'
% path)
files = sorted([
f for f in os.listdir(path)
if f.endswith('.py') and not f.startswith('_')
])
for f in files:
module = 'oeqa.controllers.' + f[:-3]
if module not in controllerslist:
controllerslist.append(module)
else:
bb.warn(
"Duplicate controller module found for %s, only one added. Layers should create unique controller module names"
% module)
for p in bbpath:
controllerpath = os.path.join(p, 'lib', 'oeqa', 'controllers')
bb.debug(2,
'Searching for target controllers in %s' % controllerpath)
if os.path.exists(controllerpath):
add_controller_list(controllerpath)
return controllerslist
def readfifo(data):
lines = data.split(b'\0')
for line in lines:
# Just skip empty commands
if not line:
continue
splitval = line.split(b' ', 1)
cmd = splitval[0].decode("utf-8")
if len(splitval) > 1:
value = splitval[1].decode("utf-8")
else:
value = ''
if cmd == 'bbplain':
bb.plain(value)
elif cmd == 'bbnote':
bb.note(value)
elif cmd == 'bbwarn':
bb.warn(value)
elif cmd == 'bberror':
bb.error(value)
elif cmd == 'bbfatal':
# The caller will call exit themselves, so bb.error() is
# what we want here rather than bb.fatal()
bb.error(value)
elif cmd == 'bbfatal_log':
bb.error(value, forcelog=True)
elif cmd == 'bbdebug':
splitval = value.split(' ', 1)
level = int(splitval[0])
value = splitval[1]
bb.debug(level, value)
else:
bb.warn("Unrecognised command '%s' on FIFO" % cmd)
def makeclient(self):
os.environ['AWS_METADATA_SERVICE_NUM_ATTEMPTS'] = '{}'.format(self.maxtries)
os.environ['AWS_METADATA_SERVICE_TIMEOUT'] = '{}'.format(self.metadata_timeout)
session = botocore.session.get_session()
self.s3client = boto3.Session(botocore_session=session).client('s3')
provider = session.get_component('credential_provider').get_provider('assume-role')
provider.cache = botocore.credentials.JSONFileCache()
bb.debug(1, "Using AWS profile: %s" % provider._profile_name)
def epochfile_write(source_date_epoch, epochfile, d):
bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
bb.utils.mkdirhier(os.path.dirname(epochfile))
tmp_file = "%s.new" % epochfile
with open(tmp_file, 'w') as f:
f.write(str(source_date_epoch))
os.rename(tmp_file, epochfile)
def handle(fn, data = bb.data.init(), include = 0):
if include:
inc_string = "including"
else:
inc_string = "reading"
init(data)
if include == 0:
bb.data.inheritFromOS(data)
oldfile = None
else:
oldfile = bb.data.getVar('FILE', data)
fn = obtain(fn, data)
bbpath = []
if not os.path.isabs(fn):
f = None
vbbpath = bb.data.getVar("BBPATH", data)
if vbbpath:
bbpath += vbbpath.split(":")
for p in bbpath:
currname = os.path.join(bb.data.expand(p, data), fn)
if os.access(currname, os.R_OK):
f = open(currname, 'r')
abs_fn = currname
debug(1, "CONF %s %s" % (inc_string, currname))
break
if f is None:
raise IOError("file '%s' not found" % fn)
else:
f = open(fn,'r')
debug(1, "CONF %s %s" % (inc_string,fn))
abs_fn = fn
if include:
bb.parse.mark_dependency(data, abs_fn)
lineno = 0
bb.data.setVar('FILE', fn, data)
while 1:
lineno = lineno + 1
s = f.readline()
if not s: break
w = s.strip()
if not w: continue # skip empty lines
s = s.rstrip()
if s[0] == '#': continue # skip comments
while s[-1] == '\\':
s2 = f.readline()[:-1].strip()
lineno = lineno + 1
s = s[:-1] + s2
feeder(lineno, s, fn, data)
if oldfile:
bb.data.setVar('FILE', oldfile, data)
return data
def _parse_path(self, regex, s):
"""
Find and group name, version and archive type in the given string s
"""
bb.debug(3, "parse_path(%s, %s)" % (regex.pattern, s))
m = regex.search(s)
if m:
bb.debug(3, "%s, %s, %s" % (m.group('name'), m.group('ver'), m.group('type')))
return (m.group('name'), m.group('ver'), m.group('type'))
return None
def _parse_path(self, regex, s):
"""
Find and group name, version and archive type in the given string s
"""
bb.debug(3, "parse_path(%s, %s)" % (regex.pattern, s))
m = regex.search(s)
if m:
bb.debug(3, "%s, %s, %s" % (m.group('name'), m.group('ver'), m.group('type')))
return (m.group('name'), m.group('ver'), m.group('type'))
return None
def applyOverrides(var, d = _data):
if not overrides:
debug(1, "OVERRIDES not defined, nothing to do")
return
val = getVar(var, d)
for o in overrides:
if var.endswith("_" + o):
l = len(o)+1
name = var[:-l]
d[name] = d[var]
def _check_latest_version_by_dir(self, dirver, package, package_regex,
current_version, ud, d):
"""
Scan every directory in order to get upstream version.
"""
version_dir = ['', '', '']
version = ['', '', '']
dirver_regex = re.compile(r"(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])*(\d+))")
s = dirver_regex.search(dirver)
if s:
version_dir[1] = s.group('ver')
else:
version_dir[1] = dirver
dirs_uri = bb.fetch.encodeurl(
[ud.type, ud.host,
ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d),
"html.parser",
parse_only=SoupStrainer("a"))
if not soup:
return version[1]
for line in soup.find_all('a', href=True):
s = dirver_regex.search(line['href'].strip("/"))
if s:
sver = s.group('ver')
# When prefix is part of the version directory it need to
# ensure that only version directory is used so remove previous
# directories if exists.
#
# Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
# result is v2.5.
spfx = s.group('pfx').split('/')[-1]
version_dir_new = ['', sver, '']
if self._vercmp(version_dir, version_dir_new) <= 0:
dirver_new = spfx + sver
path = ud.path.replace(dirver, dirver_new, True) \
.split(package)[0]
uri = bb.fetch.encodeurl(
[ud.type, ud.host, path, ud.user, ud.pswd, {}])
pupver = self._check_latest_version(
uri, package, package_regex, current_version, ud, d)
if pupver:
version[1] = pupver
version_dir = version_dir_new
return version[1]
def mkdirhier(dir):
"""Create a directory like 'mkdir -p', but does not complain if
directory already exists like os.makedirs
"""
bb.debug(3, "mkdirhier(%s)" % dir)
try:
os.makedirs(dir)
bb.debug(2, "created " + dir)
except OSError, e:
if e.errno != 17: raise e
def _get_tests_list(self, bbpath, extrapath):
testslist = []
type = self._get_test_namespace()
# This relies on lib/ under each directory in BBPATH being added to sys.path
# (as done by default in base.bbclass)
for testname in self.testsuites:
if testname != "auto":
if testname.startswith("oeqa."):
testslist.append(testname)
continue
found = False
for p in bbpath:
if os.path.exists(
os.path.join(p, extrapath, type,
testname + ".py")):
testslist.append("oeqa." + type + "." + testname)
found = True
break
elif os.path.exists(
os.path.join(p, extrapath, type,
testname.split(".")[0] + ".py")):
testslist.append("oeqa." + type + "." + testname)
found = True
break
if not found:
bb.fatal(
'Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH'
% testname)
if "auto" in self.testsuites:
def add_auto_list(path):
if not os.path.exists(os.path.join(path, '__init__.py')):
bb.fatal(
'Tests directory %s exists but is missing __init__.py'
% path)
files = sorted([
f for f in os.listdir(path)
if f.endswith('.py') and not f.startswith('_')
])
for f in files:
module = 'oeqa.' + type + '.' + f[:-3]
if module not in testslist:
testslist.append(module)
for p in bbpath:
testpath = os.path.join(p, 'lib', 'oeqa', type)
bb.debug(2, 'Searching for tests in %s' % testpath)
if os.path.exists(testpath):
add_auto_list(testpath)
return testslist
def mkdirhier(dir):
"""Create a directory like 'mkdir -p', but does not complain if
directory already exists like os.makedirs
"""
bb.debug(3, "mkdirhier(%s)" % dir)
try:
os.makedirs(dir)
bb.debug(2, "created " + dir)
except OSError, e:
if e.errno != 17: raise e
def rungitcmd(cmd,d):
bb.debug(1, "Running %s" % cmd)
# Need to export PATH as git is likely to be in metadata paths
# rather than host provided
pathcmd = 'export PATH=%s; %s' % (data.expand('${PATH}', d), cmd)
myret = os.system(pathcmd)
if myret != 0:
raise FetchError("Git: %s failed" % pathcmd)
def _check_latest_version_by_dir(self, dirver, package, package_regex,
current_version, ud, d):
"""
Scan every directory in order to get upstream version.
"""
version_dir = ['', '', '']
version = ['', '', '']
dirver_regex = re.compile("(?P<pfx>\D*)(?P<ver>(\d+[\.\-_])+(\d+))")
s = dirver_regex.search(dirver)
if s:
version_dir[1] = s.group('ver')
else:
version_dir[1] = dirver
dirs_uri = bb.fetch.encodeurl([ud.type, ud.host,
ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d), "html.parser", parse_only=SoupStrainer("a"))
if not soup:
return version[1]
for line in soup.find_all('a', href=True):
s = dirver_regex.search(line['href'].strip("/"))
if s:
sver = s.group('ver')
# When prefix is part of the version directory it need to
# ensure that only version directory is used so remove previous
# directories if exists.
#
# Example: pfx = '/dir1/dir2/v' and version = '2.5' the expected
# result is v2.5.
spfx = s.group('pfx').split('/')[-1]
version_dir_new = ['', sver, '']
if self._vercmp(version_dir, version_dir_new) <= 0:
dirver_new = spfx + sver
path = ud.path.replace(dirver, dirver_new, True) \
.split(package)[0]
uri = bb.fetch.encodeurl([ud.type, ud.host, path,
ud.user, ud.pswd, {}])
pupver = self._check_latest_version(uri,
package, package_regex, current_version, ud, d)
if pupver:
version[1] = pupver
version_dir = version_dir_new
return version[1]
def get_object_info(self, Bucket, Key, quiet=True):
if self.s3client is None:
self.makeclient()
try:
info = self.s3client.head_object(Bucket=Bucket, Key=Key)
except botocore.exceptions.ClientError as e:
err = e.response['Error']
if quiet and err['Code'] == "404":
bb.debug(2, "not found: {}/{}".format(Bucket, Key))
else:
bb.warn("{}/{}: {} {}".format(Bucket, Key, err['Code'], err['Message']))
return None
return info
def include(oldfn, fn, data = bb.data.init()):
if oldfn == fn: # prevent infinate recursion
return None
import bb
fn = bb.data.expand(fn, data)
oldfn = bb.data.expand(oldfn, data)
from bb.parse import handle
try:
ret = handle(fn, data, 1)
except IOError:
debug(2, "CONF file '%s' not found" % fn)
def init(urls = [], d = None):
if d == None:
bb.debug(2,"BUG init called with None as data object!!!")
return
for m in methods:
m.urls = []
for u in urls:
for m in methods:
m.data = d
if m.supports(u, d):
m.urls.append(u)
def exec_func_shell(func, d, runfile, cwd=None):
"""Execute a shell function from the metadata
Note on directory behavior. The 'dirs' varflag should contain a list
of the directories you need created prior to execution. The last
item in the list is where we will chdir/cd to.
"""
# Don't let the emitted shell script override PWD
d.delVarFlag('PWD', 'export')
with open(runfile, 'w') as script:
script.write(shell_trap_code())
bb.data.emit_func(func, script, d)
if bb.msg.loggerVerboseLogs:
script.write("set -x\n")
if cwd:
script.write("cd '%s'\n" % cwd)
script.write("%s\n" % func)
script.write('''
# cleanup
ret=$?
trap '' 0
exit $?
''')
os.chmod(runfile, 0775)
cmd = runfile
if d.getVarFlag(func, 'fakeroot'):
fakerootcmd = d.getVar('FAKEROOT', True)
if fakerootcmd:
cmd = [fakerootcmd, runfile]
if bb.msg.loggerDefaultVerbose:
logfile = LogTee(logger, sys.stdout)
else:
logfile = sys.stdout
bb.debug(2, "Executing shell function %s" % func)
try:
with open(os.devnull, 'r+') as stdin:
bb.process.run(cmd, shell=False, stdin=stdin, log=logfile)
except bb.process.CmdError:
logfn = d.getVar('BB_LOGFILE', True)
raise FuncFailed(func, logfn)
bb.debug(2, "Shell function %s finished" % func)
def exec_func_shell(func, d, runfile, cwd=None):
"""Execute a shell function from the metadata
Note on directory behavior. The 'dirs' varflag should contain a list
of the directories you need created prior to execution. The last
item in the list is where we will chdir/cd to.
"""
# Don't let the emitted shell script override PWD
d.delVarFlag('PWD', 'export')
with open(runfile, 'w') as script:
script.write(shell_trap_code())
bb.data.emit_func(func, script, d)
if bb.msg.loggerVerboseLogs:
script.write("set -x\n")
if cwd:
script.write("cd '%s'\n" % cwd)
script.write("%s\n" % func)
script.write('''
# cleanup
ret=$?
trap '' 0
exit $?
''')
os.chmod(runfile, 0775)
cmd = runfile
if d.getVarFlag(func, 'fakeroot'):
fakerootcmd = d.getVar('FAKEROOT', True)
if fakerootcmd:
cmd = [fakerootcmd, runfile]
if bb.msg.loggerDefaultVerbose:
logfile = LogTee(logger, sys.stdout)
else:
logfile = sys.stdout
bb.debug(2, "Executing shell function %s" % func)
try:
with open(os.devnull, 'r+') as stdin:
bb.process.run(cmd, shell=False, stdin=stdin, log=logfile)
except bb.process.CmdError:
logfn = d.getVar('BB_LOGFILE', True)
raise FuncFailed(func, logfn)
bb.debug(2, "Shell function %s finished" % func)
def download(self, ud, d):
"""Fetch url"""
logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'")
lf = bb.utils.lockfile(ud.svnlock)
try:
if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK):
svncmd = self._buildsvncommand(ud, d, "update")
logger.info("Update " + ud.url)
# We need to attempt to run svn upgrade first in case its an older working format
try:
runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir)
except FetchError:
pass
logger.debug("Running %s", svncmd)
bb.fetch2.check_network_access(d, svncmd, ud.url)
runfetchcmd(svncmd, d, workdir=ud.moddir)
else:
svncmd = self._buildsvncommand(ud, d, "fetch")
logger.info("Fetch " + ud.url)
# check out sources there
bb.utils.mkdirhier(ud.pkgdir)
logger.debug("Running %s", svncmd)
bb.fetch2.check_network_access(d, svncmd, ud.url)
runfetchcmd(svncmd, d, workdir=ud.pkgdir)
if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"):
# Warn the user if this had externals (won't catch them all)
output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir)
if output:
if "--ignore-externals" in svncmd.split():
bb.warn("%s contains svn:externals." % ud.url)
bb.warn("These should be added to the recipe SRC_URI as necessary.")
bb.warn("svn fetch has ignored externals:\n%s" % output)
bb.warn("To disable this warning add ';externals=nowarn' to the url.")
else:
bb.debug(1, "svn repository has externals:\n%s" % output)
scmdata = ud.parm.get("scmdata", "")
if scmdata == "keep":
tar_flags = ""
else:
tar_flags = "--exclude='.svn'"
# tar them up to a defined filename
runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d,
cleanup=[ud.localpath], workdir=ud.pkgdir)
finally:
bb.utils.unlockfile(lf)
def obtain(fn, data = bb.data.init()):
import sys, bb
fn = bb.data.expand(fn, data)
localfn = bb.data.expand(localpath(fn, data), data)
if localfn != fn:
dldir = bb.data.getVar('DL_DIR', data, 1)
if not dldir:
debug(1, "obtain: DL_DIR not defined")
return localfn
bb.mkdirhier(dldir)
try:
bb.fetch.init([fn])
except bb.fetch.NoMethodError:
(type, value, traceback) = sys.exc_info()
debug(1, "obtain: no method: %s" % value)
return localfn
try:
bb.fetch.go(data)
except bb.fetch.MissingParameterError:
(type, value, traceback) = sys.exc_info()
debug(1, "obtain: missing parameters: %s" % value)
return localfn
except bb.fetch.FetchError:
(type, value, traceback) = sys.exc_info()
debug(1, "obtain: failed: %s" % value)
return localfn
return localfn
def inherit(files, d):
__inherit_cache = data.getVar('__inherit_cache', d) or ""
fn = ""
lineno = 0
for f in files:
file = data.expand(f, d)
if file[0] != "/" and file[-8:] != ".bbclass":
file = os.path.join('classes', '%s.bbclass' % file)
if not file in __inherit_cache.split():
debug(2, "BB %s:%d: inheriting %s" % (fn, lineno, file))
__inherit_cache += " %s" % file
include(fn, file, d)
data.setVar('__inherit_cache', __inherit_cache, d)
def latest_versionstring(self, ud, d):
"""
Manipulate the URL and try to obtain the latest package version
sanity check to ensure same name and type.
"""
package = ud.path.split("/")[-1]
current_version = ['', d.getVar('PV'), '']
"""possible to have no version in pkg name, such as spectrum-fw"""
if not re.search(r"\d+", package):
current_version[1] = re.sub('_', '.', current_version[1])
current_version[1] = re.sub('-', '.', current_version[1])
return (current_version[1], '')
package_regex = self._init_regexes(package, ud, d)
if package_regex is None:
bb.warn("latest_versionstring: package %s don't match pattern" %
(package))
return ('', '')
bb.debug(3,
"latest_versionstring, regex: %s" % (package_regex.pattern))
uri = ""
regex_uri = d.getVar("UPSTREAM_CHECK_URI")
if not regex_uri:
path = ud.path.split(package)[0]
# search for version matches on folders inside the path, like:
# "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
dirver_regex = re.compile(
r"(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
m = dirver_regex.search(path)
if m:
pn = d.getVar('PN')
dirver = m.group('dirver')
dirver_pn_regex = re.compile(r"%s\d?" % (re.escape(pn)))
if not dirver_pn_regex.search(dirver):
return (self._check_latest_version_by_dir(
dirver, package, package_regex, current_version, ud,
d), '')
uri = bb.fetch.encodeurl(
[ud.type, ud.host, path, ud.user, ud.pswd, {}])
else:
uri = regex_uri
return (self._check_latest_version(uri, package, package_regex,
current_version, ud, d), '')
def get_source_date_epoch_from_known_files(d, sourcedir):
source_date_epoch = None
newest_file = None
known_files = set(["NEWS", "ChangeLog", "Changelog", "CHANGES"])
for file in known_files:
filepath = os.path.join(sourcedir, file)
if os.path.isfile(filepath):
mtime = int(os.lstat(filepath).st_mtime)
# There may be more than one "known_file" present, if so, use the youngest one
if not source_date_epoch or mtime > source_date_epoch:
source_date_epoch = mtime
newest_file = filepath
if newest_file:
bb.debug(1, "SOURCE_DATE_EPOCH taken from: %s" % newest_file)
return source_date_epoch
def copy_from_sysroots(pathnames, sysroots, mirrors, installdest):
'''Copy the specified files from the specified sysroots, also checking the
specified mirror patterns as alternate paths, to the specified destination.'''
import subprocess
expanded_pathnames = expand_paths(pathnames, mirrors)
searched_paths = search_sysroots(expanded_pathnames, sysroots)
for path, files in searched_paths:
if not files:
bb.debug(1, 'Failed to find `{}`'.format(path))
else:
destdir = oe.path.join(installdest, os.path.dirname(path))
bb.utils.mkdirhier(destdir)
subprocess.check_call(['cp', '-pPR'] + list(files) + [destdir + '/'])
bb.note('Copied `{}` to `{}/`'.format(', '.join(files), destdir))
def makeclient(self):
session = botocore.session.get_session()
timeout = session.get_config_variable('metadata_service_timeout')
if timeout < 10:
bb.debug(1, "increasing metadata service timeout to 10 seconds")
session.set_config_variable('metadata_service_timeout', 10)
retries = session.get_config_variable('metadata_service_num_attempts')
if retries < self.maxtries:
bb.debug(
1, "increasing metadata service retries to %d" % self.maxtries)
session.set_config_variable('metadata_service_num_attempts',
self.maxtries)
session.get_component('credential_provider').get_provider(
'assume-role').cache = botocore.credentials.JSONFileCache()
self.s3client = boto3.Session(botocore_session=session).client('s3')
def run(d, cmd, *args):
topdir = d.getVar('TOPDIR', True)
toolchain_path = d.getVar('EXTERNAL_TOOLCHAIN', True)
if toolchain_path:
target_prefix = d.getVar('EXTERNAL_TARGET_SYS', True) + '-'
path = os.path.join(toolchain_path, 'bin', target_prefix + cmd)
args = [path] + list(args)
try:
output, _ = bb.process.run(args, cwd=topdir)
except bb.process.CmdError as exc:
bb.debug(1, str(exc))
else:
return output
return 'UNKNOWN'
def run(d, cmd, *args):
topdir = d.getVar('TMPDIR', True)
toolchain_path = d.getVar('EXTERNAL_TOOLCHAIN', True)
if toolchain_path:
target_prefix = d.getVar('EXTERNAL_TARGET_SYS', True) + '-'
path = os.path.join(toolchain_path, 'bin', target_prefix + cmd)
args = [path] + list(args)
try:
output, _ = bb.process.run(args, cwd=topdir)
except bb.process.CmdError as exc:
bb.debug(1, str(exc))
else:
return output
return 'UNKNOWN'
def copy_from_sysroots(pathnames, sysroots, mirrors, installdest):
'''Copy the specified files from the specified sysroots, also checking the
specified mirror patterns as alternate paths, to the specified destination.'''
import subprocess
expanded_pathnames = expand_paths(pathnames, mirrors)
searched_paths = search_sysroots(expanded_pathnames, sysroots)
for path, files in searched_paths:
if not files:
bb.debug(1, 'Failed to find `{}`'.format(path))
else:
destdir = oe.path.join(installdest, os.path.dirname(path))
bb.utils.mkdirhier(destdir)
subprocess.check_call(['cp', '-pPR'] + list(files) +
[destdir + '/'])
bb.note('Copied `{}` to `{}/`'.format(', '.join(files), destdir))
def latest_versionstring(self, ud, d):
"""
Manipulate the URL and try to obtain the latest package version
sanity check to ensure same name and type.
"""
package = ud.path.split("/")[-1]
current_version = ["", d.getVar("PV", True), ""]
"""possible to have no version in pkg name, such as spectrum-fw"""
if not re.search("\d+", package):
current_version[1] = re.sub("_", ".", current_version[1])
current_version[1] = re.sub("-", ".", current_version[1])
return (current_version[1], "")
package_regex = self._init_regexes(package, ud, d)
if package_regex is None:
bb.warn("latest_versionstring: package %s don't match pattern" % (package))
return ("", "")
bb.debug(3, "latest_versionstring, regex: %s" % (package_regex.pattern))
uri = ""
regex_uri = d.getVar("UPSTREAM_CHECK_URI", True)
if not regex_uri:
path = ud.path.split(package)[0]
# search for version matches on folders inside the path, like:
# "5.7" in http://download.gnome.org/sources/${PN}/5.7/${PN}-${PV}.tar.gz
dirver_regex = re.compile("(?P<dirver>[^/]*(\d+\.)*\d+([-_]r\d+)*)/")
m = dirver_regex.search(path)
if m:
pn = d.getVar("PN", True)
dirver = m.group("dirver")
dirver_pn_regex = re.compile("%s\d?" % (re.escape(pn)))
if not dirver_pn_regex.search(dirver):
return (
self._check_latest_version_by_dir(dirver, package, package_regex, current_version, ud, d),
"",
)
uri = bb.fetch.encodeurl([ud.type, ud.host, path, ud.user, ud.pswd, {}])
else:
uri = regex_uri
return (self._check_latest_version(uri, package, package_regex, current_version, ud, d), "")
def _check_latest_version_by_dir(self, dirver, package, package_regex,
current_version, ud, d):
"""
Scan every directory in order to get upstream version.
"""
version_dir = ['', '', '']
version = ['', '', '']
dirver_regex = re.compile("(\D*)((\d+[\.\-_])+(\d+))")
s = dirver_regex.search(dirver)
if s:
version_dir[1] = s.group(2)
else:
version_dir[1] = dirver
dirs_uri = bb.fetch.encodeurl(
[ud.type, ud.host,
ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d),
"html.parser",
parse_only=SoupStrainer("a"))
if not soup:
return version[1]
for line in soup.find_all('a', href=True):
s = dirver_regex.search(line['href'].strip("/"))
if s:
version_dir_new = ['', s.group(2), '']
if self._vercmp(version_dir, version_dir_new) <= 0:
dirver_new = s.group(1) + s.group(2)
path = ud.path.replace(dirver, dirver_new, True) \
.split(package)[0]
uri = bb.fetch.encodeurl(
[ud.type, ud.host, path, ud.user, ud.pswd, {}])
pupver = self._check_latest_version(
uri, package, package_regex, current_version, ud, d)
if pupver:
version[1] = pupver
version_dir = version_dir_new
return version[1]
def get_object_info(self, Bucket, Key, quiet=True):
if self.s3client is None:
self.makeclient()
for attempt in range(10):
try:
info = self.s3client.head_object(Bucket=Bucket, Key=Key)
except botocore.exceptions.NoCredentialsError:
s3retry_wait(attempt)
continue
except botocore.exceptions.ClientError as e:
err = e.response['Error']
if quiet and err['Code'] == "404":
bb.debug(2, "not found: {}/{}".format(Bucket, Key))
else:
bb.warn("{}/{}: {} {}".format(Bucket, Key, err['Code'], err['Message']))
return None
return info
bb.warn("{}/{}: credentials error on get_object_info for 10 attempts".format(Bucket, Key))
return None
def exec_func_python(func, d, runfile, cwd=None, pythonexception=False):
"""Execute a python BB 'function'"""
code = _functionfmt.format(function=func)
bb.utils.mkdirhier(os.path.dirname(runfile))
with open(runfile, 'w') as script:
bb.data.emit_func_python(func, script, d)
if cwd:
try:
olddir = os.getcwd()
except OSError:
olddir = None
os.chdir(cwd)
bb.debug(2, "Executing python function %s" % func)
try:
text = "def %s(d):\n%s" % (func, d.getVar(func, False))
fn = d.getVarFlag(func, "filename", False)
lineno = int(d.getVarFlag(func, "lineno", False))
bb.methodpool.insert_method(func, text, fn, lineno - 1)
comp = utils.better_compile(code, func,
"exec_python_func() autogenerated")
utils.better_exec(comp, {"d": d},
code,
"exec_python_func() autogenerated",
pythonexception=pythonexception)
except (bb.parse.SkipRecipe, bb.build.FuncFailed):
raise
except:
if pythonexception:
raise
raise FuncFailed(func, None)
finally:
bb.debug(2, "Python function %s finished" % func)
if cwd and olddir:
try:
os.chdir(olddir)
except OSError:
pass
def arch_config_sub(d, arch):
try:
config_sub_cache = globals()['config_sub_cache']
except KeyError:
config_sub_cache = {}
globals()['config_sub_cache'] = config_sub_cache
try:
canonical_arch = config_sub_cache[arch]
except KeyError:
script = arch_find_script(d, 'config.sub')
try:
bb.debug("%s %s"%(script, arch))
canonical_arch = os.popen("%s %s"%(script, arch)).readline().strip()
config_sub_cache[arch] = canonical_arch
except OSError, e:
bb.error("config.sub(%s) failed: %s"%(arch, e))
return arch
def prserv_import_db(d,
filter_version=None,
filter_pkgarch=None,
filter_checksum=None):
if d.getVar('USE_PR_SERV', True) != "1":
bb.error("Not using network based PR service")
return None
conn = d.getVar("__PRSERV_CONN", True)
if conn is None:
conn = prserv_make_conn(d)
if conn is None:
bb.error("Making connection failed to remote PR service")
return None
#get the entry values
imported = []
prefix = "PRAUTO$"
for v in d.keys():
if v.startswith(prefix):
(remain, sep, checksum) = v.rpartition('$')
(remain, sep, pkgarch) = remain.rpartition('$')
(remain, sep, version) = remain.rpartition('$')
if (remain + '$' != prefix) or \
(filter_version and filter_version != version) or \
(filter_pkgarch and filter_pkgarch != pkgarch) or \
(filter_checksum and filter_checksum != checksum):
continue
try:
value = int(
d.getVar(
remain + '$' + version + '$' + pkgarch + '$' +
checksum, True))
except BaseException as exc:
bb.debug("Not valid value of %s:%s" % (v, str(exc)))
continue
ret = conn.importone(version, pkgarch, checksum, value)
if ret != value:
bb.error(
"importing(%s,%s,%s,%d) failed. DB may have larger value %d"
% (version, pkgarch, checksum, value, ret))
else:
imported.append((version, pkgarch, checksum, value))
return imported
def include(oldfn, fn, data = bb.data.init(), error_out = False):
"""
error_out If True a ParseError will be reaised if the to be included
"""
if oldfn == fn: # prevent infinate recursion
return None
import bb
fn = bb.data.expand(fn, data)
oldfn = bb.data.expand(oldfn, data)
from bb.parse import handle
try:
ret = handle(fn, data, True)
except IOError:
if error_out:
raise ParseError("Could not include required file %(fn)s" % vars() )
debug(2, "CONF file '%s' not found" % fn)
def check_for_tarball(d, tarfn, dldir, date):
"""
Check for a local copy then check the tarball stash.
Both checks are skipped if date == 'now'.
d Is a bb.data instance
tarfn is the name of the tarball
date is the SRCDATE
"""
if "now" != date:
dl = os.path.join(dldir, tarfn)
if os.access(dl, os.R_OK):
bb.debug(1, "%s already exists, skipping checkout." % tarfn)
return True
# try to use the tarball stash
if Fetch.try_mirror(d, tarfn):
return True
return False
def arch_config_sub(d, arch):
try:
config_sub_cache = globals()['config_sub_cache']
except KeyError:
config_sub_cache = {}
globals()['config_sub_cache'] = config_sub_cache
try:
canonical_arch = config_sub_cache[arch]
except KeyError:
script = arch_find_script(d, 'config.sub')
try:
bb.debug("%s %s"%(script, arch))
canonical_arch = os.popen("%s %s"%(script, arch)).readline().strip()
config_sub_cache[arch] = canonical_arch
except OSError, e:
bb.error("config.sub(%s) failed: %s"%(arch, e))
return arch
def _get_tests_list(self, bbpath, extrapath):
testslist = []
type = self._get_test_namespace()
# This relies on lib/ under each directory in BBPATH being added to sys.path
# (as done by default in base.bbclass)
for testname in self.testsuites:
if testname != "auto":
if testname.startswith("oeqa."):
testslist.append(testname)
continue
found = False
for p in bbpath:
if os.path.exists(os.path.join(p, extrapath, type, testname + ".py")):
testslist.append("oeqa." + type + "." + testname)
found = True
break
elif os.path.exists(os.path.join(p, extrapath, type, testname.split(".")[0] + ".py")):
testslist.append("oeqa." + type + "." + testname)
found = True
break
if not found:
bb.fatal('Test %s specified in TEST_SUITES could not be found in lib/oeqa/runtime under BBPATH' % testname)
if "auto" in self.testsuites:
def add_auto_list(path):
if not os.path.exists(os.path.join(path, '__init__.py')):
bb.fatal('Tests directory %s exists but is missing __init__.py' % path)
files = sorted([f for f in os.listdir(path) if f.endswith('.py') and not f.startswith('_')])
for f in files:
module = 'oeqa.' + type + '.' + f[:-3]
if module not in testslist:
testslist.append(module)
for p in bbpath:
testpath = os.path.join(p, 'lib', 'oeqa', type)
bb.debug(2, 'Searching for tests in %s' % testpath)
if os.path.exists(testpath):
add_auto_list(testpath)
return testslist
def _check_latest_version_by_dir(self, dirver, package, package_regex,
current_version, ud, d):
"""
Scan every directory in order to get upstream version.
"""
version_dir = ['', '', '']
version = ['', '', '']
dirver_regex = re.compile("(\D*)((\d+[\.-_])+(\d+))")
s = dirver_regex.search(dirver)
if s:
version_dir[1] = s.group(2)
else:
version_dir[1] = dirver
dirs_uri = bb.fetch.encodeurl([ud.type, ud.host,
ud.path.split(dirver)[0], ud.user, ud.pswd, {}])
bb.debug(3, "DirURL: %s, %s" % (dirs_uri, package))
soup = BeautifulSoup(self._fetch_index(dirs_uri, ud, d))
if not soup:
return version[1]
for line in soup.find_all('a', href=True):
s = dirver_regex.search(line['href'].strip("/"))
if s:
version_dir_new = ['', s.group(2), '']
if self._vercmp(version_dir, version_dir_new) <= 0:
dirver_new = s.group(1) + s.group(2)
path = ud.path.replace(dirver, dirver_new, True) \
.split(package)[0]
uri = bb.fetch.encodeurl([ud.type, ud.host, path,
ud.user, ud.pswd, {}])
pupver = self._check_latest_version(uri,
package, package_regex, current_version, ud, d)
if pupver:
version[1] = pupver
version_dir = version_dir_new
return version[1]
def filter_environment(good_vars):
"""
Create a pristine environment for bitbake. This will remove variables that
are not known and may influence the build in a negative way.
"""
import bb
removed_vars = []
for key in os.environ.keys():
if key in good_vars:
continue
removed_vars.append(key)
os.unsetenv(key)
del os.environ[key]
if len(removed_vars):
bb.debug(1, "Removed the following variables from the environment:", ",".join(removed_vars))
return removed_vars
def epochfile_read(epochfile, d):
cached, efile = d.getVar('__CACHED_SOURCE_DATE_EPOCH') or (None, None)
if cached and efile == epochfile:
return cached
if cached and epochfile != efile:
bb.debug(1, "Epoch file changed from %s to %s" % (efile, epochfile))
source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK'))
try:
with open(epochfile, 'r') as f:
s = f.read()
try:
source_date_epoch = int(s)
except ValueError:
bb.warn(
"SOURCE_DATE_EPOCH value '%s' is invalid. Reverting to SOURCE_DATE_EPOCH_FALLBACK"
% s)
source_date_epoch = int(d.getVar('SOURCE_DATE_EPOCH_FALLBACK'))
bb.debug(1, "SOURCE_DATE_EPOCH: %d" % source_date_epoch)
except FileNotFoundError:
bb.debug(
1, "Cannot find %s. SOURCE_DATE_EPOCH will default to %d" %
(epochfile, source_date_epoch))
d.setVar('__CACHED_SOURCE_DATE_EPOCH', (str(source_date_epoch), epochfile))
return str(source_date_epoch)
