def create_socket(url, d):
import urllib
socket = None
try:
export_proxies(d)
socket = urllib.request.urlopen(url)
except:
bb.warn("distro_check: create_socket url %s can't access" % url)
return socket
def create_socket(url, d):
import urllib
socket = None
try:
export_proxies(d)
socket = urllib.request.urlopen(url)
except:
bb.warn("distro_check: create_socket url %s can't access" % url)
return socket
def checkstatus(self, fetch, ud, d, try_again=True):
import urllib.request, urllib.error, urllib.parse, socket, http.client
from urllib.response import addinfourl
from bb.fetch2 import FetchConnectionCache
class HTTPConnectionCache(http.client.HTTPConnection):
if fetch.connection_cache:
def connect(self):
"""Connect to the host and port specified in __init__."""
sock = fetch.connection_cache.get_connection(self.host, self.port)
if sock:
self.sock = sock
else:
self.sock = socket.create_connection((self.host, self.port),
self.timeout, self.source_address)
fetch.connection_cache.add_connection(self.host, self.port, self.sock)
if self._tunnel_host:
self._tunnel()
class CacheHTTPHandler(urllib.request.HTTPHandler):
def http_open(self, req):
return self.do_open(HTTPConnectionCache, req)
def do_open(self, http_class, req):
"""Return an addinfourl object for the request, using http_class.
http_class must implement the HTTPConnection API from httplib.
The addinfourl return value is a file-like object. It also
has methods and attributes including:
- info(): return a mimetools.Message object for the headers
- geturl(): return the original request URL
- code: HTTP status code
"""
host = req.host
if not host:
raise urlllib2.URLError('no host given')
h = http_class(host, timeout=req.timeout) # will parse host:port
h.set_debuglevel(self._debuglevel)
headers = dict(req.unredirected_hdrs)
headers.update(dict((k, v) for k, v in list(req.headers.items())
if k not in headers))
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
# Don't close connection when connection_cache is enabled,
if fetch.connection_cache is None:
headers["Connection"] = "close"
else:
headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
headers = dict(
(name.title(), val) for name, val in list(headers.items()))
if req._tunnel_host:
tunnel_headers = {}
proxy_auth_hdr = "Proxy-Authorization"
if proxy_auth_hdr in headers:
tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
# Proxy-Authorization should not be sent to origin
# server.
del headers[proxy_auth_hdr]
h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
try:
h.request(req.get_method(), req.selector, req.data, headers)
except socket.error as err: # XXX what error?
# Don't close connection when cache is enabled.
# Instead, try to detect connections that are no longer
# usable (for example, closed unexpectedly) and remove
# them from the cache.
if fetch.connection_cache is None:
h.close()
elif isinstance(err, OSError) and err.errno == errno.EBADF:
# This happens when the server closes the connection despite the Keep-Alive.
# Apparently urllib then uses the file descriptor, expecting it to be
# connected, when in reality the connection is already gone.
# We let the request fail and expect it to be
# tried once more ("try_again" in check_status()),
# with the dead connection removed from the cache.
# If it still fails, we give up, which can happend for bad
# HTTP proxy settings.
fetch.connection_cache.remove_connection(h.host, h.port)
raise urllib.error.URLError(err)
else:
try:
r = h.getresponse(buffering=True)
except TypeError: # buffering kw not supported
r = h.getresponse()
# Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly.
# Wrap the HTTPResponse object in socket's file object adapter
# for Windows. That adapter calls recv(), so delegate recv()
# to read(). This weird wrapping allows the returned object to
# have readline() and readlines() methods.
# XXX It might be better to extract the read buffering code
# out of socket._fileobject() and into a base class.
r.recv = r.read
# no data, just have to read
r.read()
class fp_dummy(object):
def read(self):
return ""
def readline(self):
return ""
def close(self):
pass
resp = addinfourl(fp_dummy(), r.msg, req.get_full_url())
resp.code = r.status
resp.msg = r.reason
# Close connection when server request it.
if fetch.connection_cache is not None:
if 'Connection' in r.msg and r.msg['Connection'] == 'close':
fetch.connection_cache.remove_connection(h.host, h.port)
return resp
class HTTPMethodFallback(urllib.request.BaseHandler):
"""
Fallback to GET if HEAD is not allowed (405 HTTP error)
"""
def http_error_405(self, req, fp, code, msg, headers):
fp.read()
fp.close()
newheaders = dict((k,v) for k,v in list(req.headers.items())
if k.lower() not in ("content-length", "content-type"))
return self.parent.open(urllib.request.Request(req.get_full_url(),
headers=newheaders,
origin_req_host=req.origin_req_host,
unverifiable=True))
"""
Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
Forbidden when they actually mean 405 Method Not Allowed.
"""
http_error_403 = http_error_405
class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
"""
urllib2.HTTPRedirectHandler resets the method to GET on redirect,
when we want to follow redirects using the original method.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
newreq.get_method = lambda: req.get_method()
return newreq
exported_proxies = export_proxies(d)
handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
if export_proxies:
handlers.append(urllib.request.ProxyHandler())
handlers.append(CacheHTTPHandler())
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
# see PEP-0476, this causes verification errors on some https servers
# so disable by default.
import ssl
if hasattr(ssl, '_create_unverified_context'):
handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
opener = urllib.request.build_opener(*handlers)
try:
uri = ud.url.split(";")[0]
r = urllib.request.Request(uri)
r.get_method = lambda: "HEAD"
# Some servers (FusionForge, as used on Alioth) require that the
# optional Accept header is set.
r.add_header("Accept", "*/*")
def add_basic_auth(login_str, request):
'''Adds Basic auth to http request, pass in login:password as string'''
import base64
encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
authheader = "Basic %s" % encodeuser
r.add_header("Authorization", authheader)
if ud.user:
add_basic_auth(ud.user, r)
try:
import netrc, urllib.parse
n = netrc.netrc()
login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
add_basic_auth("%s:%s" % (login, password), r)
except (TypeError, ImportError, IOError, netrc.NetrcParseError):
pass
opener.open(r)
except urllib.error.URLError as e:
if try_again:
logger.debug(2, "checkstatus: trying again")
return self.checkstatus(fetch, ud, d, False)
else:
# debug for now to avoid spamming the logs in e.g. remote sstate searches
logger.debug(2, "checkstatus() urlopen failed: %s" % e)
return False
return True
def checkstatus(self, fetch, ud, d, try_again=True):
import urllib.request, urllib.error, urllib.parse, socket, http.client
from urllib.response import addinfourl
from bb.fetch2 import FetchConnectionCache
class HTTPConnectionCache(http.client.HTTPConnection):
if fetch.connection_cache:
def connect(self):
"""Connect to the host and port specified in __init__."""
sock = fetch.connection_cache.get_connection(self.host, self.port)
if sock:
self.sock = sock
else:
self.sock = socket.create_connection((self.host, self.port),
self.timeout, self.source_address)
fetch.connection_cache.add_connection(self.host, self.port, self.sock)
if self._tunnel_host:
self._tunnel()
class CacheHTTPHandler(urllib.request.HTTPHandler):
def http_open(self, req):
return self.do_open(HTTPConnectionCache, req)
def do_open(self, http_class, req):
"""Return an addinfourl object for the request, using http_class.
http_class must implement the HTTPConnection API from httplib.
The addinfourl return value is a file-like object. It also
has methods and attributes including:
- info(): return a mimetools.Message object for the headers
- geturl(): return the original request URL
- code: HTTP status code
"""
host = req.host
if not host:
raise urlllib2.URLError('no host given')
h = http_class(host, timeout=req.timeout) # will parse host:port
h.set_debuglevel(self._debuglevel)
headers = dict(req.unredirected_hdrs)
headers.update(dict((k, v) for k, v in list(req.headers.items())
if k not in headers))
# We want to make an HTTP/1.1 request, but the addinfourl
# class isn't prepared to deal with a persistent connection.
# It will try to read all remaining data from the socket,
# which will block while the server waits for the next request.
# So make sure the connection gets closed after the (only)
# request.
# Don't close connection when connection_cache is enabled,
if fetch.connection_cache is None:
headers["Connection"] = "close"
else:
headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0
headers = dict(
(name.title(), val) for name, val in list(headers.items()))
if req._tunnel_host:
tunnel_headers = {}
proxy_auth_hdr = "Proxy-Authorization"
if proxy_auth_hdr in headers:
tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr]
# Proxy-Authorization should not be sent to origin
# server.
del headers[proxy_auth_hdr]
h.set_tunnel(req._tunnel_host, headers=tunnel_headers)
try:
h.request(req.get_method(), req.selector, req.data, headers)
except socket.error as err: # XXX what error?
# Don't close connection when cache is enabled.
if fetch.connection_cache is None:
h.close()
raise urllib.error.URLError(err)
else:
try:
r = h.getresponse(buffering=True)
except TypeError: # buffering kw not supported
r = h.getresponse()
# Pick apart the HTTPResponse object to get the addinfourl
# object initialized properly.
# Wrap the HTTPResponse object in socket's file object adapter
# for Windows. That adapter calls recv(), so delegate recv()
# to read(). This weird wrapping allows the returned object to
# have readline() and readlines() methods.
# XXX It might be better to extract the read buffering code
# out of socket._fileobject() and into a base class.
r.recv = r.read
# no data, just have to read
r.read()
class fp_dummy(object):
def read(self):
return ""
def readline(self):
return ""
def close(self):
pass
resp = addinfourl(fp_dummy(), r.msg, req.get_full_url())
resp.code = r.status
resp.msg = r.reason
# Close connection when server request it.
if fetch.connection_cache is not None:
if 'Connection' in r.msg and r.msg['Connection'] == 'close':
fetch.connection_cache.remove_connection(h.host, h.port)
return resp
class HTTPMethodFallback(urllib.request.BaseHandler):
"""
Fallback to GET if HEAD is not allowed (405 HTTP error)
"""
def http_error_405(self, req, fp, code, msg, headers):
fp.read()
fp.close()
newheaders = dict((k,v) for k,v in list(req.headers.items())
if k.lower() not in ("content-length", "content-type"))
return self.parent.open(urllib.request.Request(req.get_full_url(),
headers=newheaders,
origin_req_host=req.origin_req_host,
unverifiable=True))
"""
Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403
Forbidden when they actually mean 405 Method Not Allowed.
"""
http_error_403 = http_error_405
"""
Some servers (e.g. FusionForge) returns 406 Not Acceptable when they
actually mean 405 Method Not Allowed.
"""
http_error_406 = http_error_405
class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler):
"""
urllib2.HTTPRedirectHandler resets the method to GET on redirect,
when we want to follow redirects using the original method.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
newreq.get_method = lambda: req.get_method()
return newreq
exported_proxies = export_proxies(d)
handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
if export_proxies:
handlers.append(urllib.request.ProxyHandler())
handlers.append(CacheHTTPHandler())
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
# see PEP-0476, this causes verification errors on some https servers
# so disable by default.
import ssl
if hasattr(ssl, '_create_unverified_context'):
handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context()))
opener = urllib.request.build_opener(*handlers)
try:
uri = ud.url.split(";")[0]
r = urllib.request.Request(uri)
r.get_method = lambda: "HEAD"
def add_basic_auth(login_str, request):
'''Adds Basic auth to http request, pass in login:password as string'''
import base64
encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8")
authheader = "Basic %s" % encodeuser
r.add_header("Authorization", authheader)
if ud.user:
add_basic_auth(ud.user, r)
try:
import netrc, urllib.parse
n = netrc.netrc()
login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname)
add_basic_auth("%s:%s" % (login, password), r)
except (TypeError, ImportError, IOError, netrc.NetrcParseError):
pass
opener.open(r)
except urllib.error.URLError as e:
if try_again:
logger.debug(2, "checkstatus: trying again")
return self.checkstatus(fetch, ud, d, False)
else:
# debug for now to avoid spamming the logs in e.g. remote sstate searches
logger.debug(2, "checkstatus() urlopen failed: %s" % e)
return False
return True
def download_old_versions(d):
"""
Download the necessary information from the update repo that is needed
to build updates in that update stream. This can run in parallel to
a normal build and thus is not on the critical path.
"""
content_url = d.getVar('SWUPD_CONTENT_BUILD_URL', True)
version_url = d.getVar('SWUPD_VERSION_BUILD_URL', True)
current_format = int(d.getVar('SWUPD_FORMAT', True))
deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True)
www_dir = os.path.join(deploy_dir, 'www')
if not content_url or not version_url:
bb.warn(
'SWUPD_CONTENT_BUILD_URL and/or SWUPD_VERSION_BUILD_URL not set, skipping download of old versions for the initial build of a swupd update stream.'
)
return
# Avoid double // in path. At least twisted is sensitive to that.
content_url = content_url.rstrip('/')
# Set up env variables with proxy information for use in urllib.
export_proxies(d)
# Find latest version for each of the older formats.
# For now we ignore the released milestones and go
# directly to the URL with all builds. The information
# about milestones may be relevant for determining
# how format changes need to be handled.
latest_versions = {}
for format in range(3, current_format + 1):
try:
url = '%s/version/format%d/latest' % (content_url, format)
response = urllib.request.urlopen(url)
version = int(response.read())
latest_versions[format] = version
formatdir = os.path.join(www_dir, 'version', 'format%d' % format)
bb.utils.mkdirhier(formatdir)
with open(os.path.join(formatdir, 'latest'), 'w') as latest:
latest.write(str(version))
except urllib.error.HTTPError as http_error:
if http_error.code == 404:
bb.debug(1, '%s does not exist, skipping that format' % url)
else:
raise
except urllib.error.URLError as url_error:
# Happens for file:// URLs.
if isinstance(url_error.reason,
OSError) and url_error.reason.errno == errno.ENOENT:
bb.debug(1, '%s does not exist, skipping that format' % url)
else:
raise
# Now get the Manifests of the latest versions and the
# versions we are supposed to provide a delta for, as a starting point.
# In addition, we also need Manifests that provide files reused by
# these initial set of Manifests or get referenced by them.
#
# There's no integrity checking for the files. bsdtar is
# expected to detect corrupted archives and https is expected
# to protect against man-in-the-middle attacks.
pending_versions = set(latest_versions.values())
pending_versions.update(
[int(x) for x in d.getVar('SWUPD_DELTAPACK_VERSIONS', True).split()])
fetched_versions = set([0])
while pending_versions:
version = pending_versions.pop()
sub_versions = set()
sub_versions.update(
download_manifests(content_url, version, 'MoM',
os.path.join(www_dir, str(version))))
sub_versions.update(
download_manifests(content_url, version, 'full',
os.path.join(www_dir, str(version))))
fetched_versions.add(version)
pending_versions.update(sub_versions.difference(fetched_versions))
latest_version_file = os.path.join(deploy_dir, 'image', 'latest.version')
if not os.path.exists(latest_version_file):
# We located information about latest version from online www update repo.
# Now use that to determine what we are updating from. Doing this here
# instead of swupd-image.bbclass has the advantage that we can do some
# sanity checking very early in a build.
#
# Building a proper update makes swupd_create_fullfiles
# a lot faster because it allows reusing existing, unmodified files.
# Saves a lot of space, too, because the new Manifest files then merely
# point to the older version (no entry in ${DEPLOY_DIR_SWUPD}/www/${OS_VERSION}/files,
# not even a link).
if not latest_versions:
bb.fatal(
"%s does not exist and no information was found under SWUPD_CONTENT_BUILD_URL %s, cannot proceed without information about the previous build. When building the initial version, unset SWUPD_VERSION_BUILD_URL and SWUPD_CONTENT_BUILD_URL to proceed."
% (latest_version_file, content_url))
latest = sorted(latest_versions.values())[-1]
bb.debug(2, "Setting %d in latest.version file" % latest)
with open(latest_version_file, 'w') as f:
f.write(str(latest))
actually mean 405 Method Not Allowed.
"""
http_error_406 = http_error_405
class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler):
"""
urllib2.HTTPRedirectHandler resets the method to GET on redirect,
when we want to follow redirects using the original method.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl)
newreq.get_method = lambda: req.get_method()
return newreq
exported_proxies = export_proxies(d)
handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
if export_proxies:
handlers.append(urllib2.ProxyHandler())
handlers.append(CacheHTTPHandler())
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
# see PEP-0476, this causes verification errors on some https servers
# so disable by default.
import ssl
if hasattr(ssl, "_create_unverified_context"):
handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context()))
opener = urllib2.build_opener(*handlers)
try:
def run(self, d):
import os
import subprocess
import json
import logging
from bb.utils import export_proxies
from oeqa.utils import make_logger_bitbake_compatible
pn = d.getVar("PN")
logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
# sdk use network for download projects for build
export_proxies(d)
tcname = self.get_tcname(d)
if not os.path.exists(tcname):
bb.fatal(
"The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ."
% tcname)
tdname = d.expand(
"${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
test_data = json.load(open(tdname, "r"))
target_pkg_manifest = self.context_executor_class._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
host_pkg_manifest = self.context_executor_class._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
processes = d.getVar("TESTIMAGE_NUMBER_THREADS") or d.getVar(
"BB_NUMBER_THREADS")
if processes:
try:
import testtools, subunit
except ImportError:
bb.warn(
"Failed to import testtools or subunit, the testcases will run serially"
)
processes = None
sdk_dir = d.expand("${WORKDIR}/testimage-sdk/")
bb.utils.remove(sdk_dir, True)
bb.utils.mkdirhier(sdk_dir)
context_args = self.setup_context(d)
self.extract_sdk(tcname, sdk_dir, d)
fail = False
sdk_envs = self.context_executor_class._get_sdk_environs(sdk_dir)
for s in sdk_envs:
sdk_env = sdk_envs[s]
bb.plain("SDK testing environment: %s" % s)
tc = self.context_class(td=test_data,
logger=logger,
sdk_dir=sdk_dir,
sdk_env=sdk_env,
target_pkg_manifest=target_pkg_manifest,
host_pkg_manifest=host_pkg_manifest,
**context_args)
try:
tc.loadTests(self.context_executor_class.default_cases)
except Exception as e:
import traceback
bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
if processes:
result = tc.runTests(processes=int(processes))
else:
result = tc.runTests()
component = "%s %s" % (pn, self.context_executor_class.name)
context_msg = "%s:%s" % (os.path.basename(tcname),
os.path.basename(sdk_env))
configuration = self.get_sdk_configuration(d, self.test_type)
result.logDetails(self.get_sdk_json_result_dir(d), configuration,
self.get_sdk_result_id(configuration))
result.logSummary(component, context_msg)
if not result.wasSuccessful():
fail = True
if fail:
bb.fatal("%s - FAILED - check the task log and the commands log" %
pn)
def create_socket(url, d):
import urllib
from bb.utils import export_proxies
export_proxies(d)
return urllib.request.urlopen(url)
actually mean 405 Method Not Allowed.
"""
http_error_406 = http_error_405
class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler):
"""
urllib2.HTTPRedirectHandler resets the method to GET on redirect,
when we want to follow redirects using the original method.
"""
def redirect_request(self, req, fp, code, msg, headers, newurl):
newreq = urllib2.HTTPRedirectHandler.redirect_request(
self, req, fp, code, msg, headers, newurl)
newreq.get_method = lambda: req.get_method()
return newreq
exported_proxies = export_proxies(d)
handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback]
if export_proxies:
handlers.append(urllib2.ProxyHandler())
handlers.append(CacheHTTPHandler())
# XXX: Since Python 2.7.9 ssl cert validation is enabled by default
# see PEP-0476, this causes verification errors on some https servers
# so disable by default.
import ssl
if hasattr(ssl, '_create_unverified_context'):
handlers.append(
urllib2.HTTPSHandler(context=ssl._create_unverified_context()))
opener = urllib2.build_opener(*handlers)
try:
def create_socket(url, d):
import urllib
from bb.utils import export_proxies
export_proxies(d)
return urllib.request.urlopen(url)
def run(self, d):
import os
import json
import subprocess
import logging
from bb.utils import export_proxies
from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak
from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
pn = d.getVar("PN")
logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
# extensible sdk use network
export_proxies(d)
subprocesstweak.errors_have_output()
# We need the original PATH for testing the eSDK, not with our manipulations
os.environ['PATH'] = d.getVar("BB_ORIGENV", False).getVar("PATH")
tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh")
if not os.path.exists(tcname):
bb.fatal("The toolchain ext %s is not built. Build it before running the" \
" tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname)
tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json")
test_data = json.load(open(tdname, "r"))
target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"))
host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"))
sdk_dir = d.expand("${WORKDIR}/testsdkext/")
bb.utils.remove(sdk_dir, True)
bb.utils.mkdirhier(sdk_dir)
try:
subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True)
except subprocess.CalledProcessError as e:
msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8")
logfn = os.path.join(sdk_dir, 'preparing_build_system.log')
if os.path.exists(logfn):
msg += '\n\nContents of preparing_build_system.log:\n'
with open(logfn, 'r') as f:
for line in f:
msg += line
bb.fatal(msg)
fail = False
sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir)
for s in sdk_envs:
bb.plain("Extensible SDK testing environment: %s" % s)
sdk_env = sdk_envs[s]
# Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache
# and we don't spend hours downloading kernels for the kernel module test
# Abuse auto.conf since local.conf would be overwritten by the SDK
with open(os.path.join(sdk_dir, 'conf', 'auto.conf'), 'a+') as f:
f.write('SSTATE_MIRRORS += " \\n file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR'))
f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR'))
f.write('INHERIT += "own-mirrors"\n')
f.write('PREMIRRORS:prepend = " git://git.yoctoproject.org/.* git://%s/git2/git.yoctoproject.org.BASENAME \\n "\n' % test_data.get('DL_DIR'))
# We need to do this in case we have a minimal SDK
subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % \
sdk_env, cwd=sdk_dir, shell=True, stderr=subprocess.STDOUT)
tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir,
sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
host_pkg_manifest=host_pkg_manifest)
try:
tc.loadTests(OESDKExtTestContextExecutor.default_cases)
except Exception as e:
import traceback
bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
result = tc.runTests()
component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
configuration = self.get_sdk_configuration(d, 'sdkext')
result.logDetails(self.get_sdk_json_result_dir(d),
configuration,
self.get_sdk_result_id(configuration))
result.logSummary(component, context_msg)
if not result.wasSuccessful():
fail = True
# Clean the workspace/sources to avoid `devtool add' failure because of non-empty source directory
bb.utils.remove(sdk_dir+'workspace/sources', True)
if fail:
bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
def run(self, d):
import os
import json
import subprocess
import logging
from bb.utils import export_proxies
from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak
from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor
pn = d.getVar("PN")
logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
# extensible sdk use network
export_proxies(d)
subprocesstweak.errors_have_output()
# extensible sdk can be contaminated if native programs are
# in PATH, i.e. use perl-native instead of eSDK one.
paths_to_avoid = [d.getVar('STAGING_DIR'),
d.getVar('BASE_WORKDIR')]
os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid)
tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh")
if not os.path.exists(tcname):
bb.fatal("The toolchain ext %s is not built. Build it before running the" \
" tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname)
tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json")
test_data = json.load(open(tdname, "r"))
target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest"))
host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest"))
sdk_dir = d.expand("${WORKDIR}/testsdkext/")
bb.utils.remove(sdk_dir, True)
bb.utils.mkdirhier(sdk_dir)
try:
subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True)
except subprocess.CalledProcessError as e:
msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8")
logfn = os.path.join(sdk_dir, 'preparing_build_system.log')
if os.path.exists(logfn):
msg += '\n\nContents of preparing_build_system.log:\n'
with open(logfn, 'r') as f:
for line in f:
msg += line
bb.fatal(msg)
fail = False
sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir)
for s in sdk_envs:
bb.plain("Extensible SDK testing environment: %s" % s)
sdk_env = sdk_envs[s]
# Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache
# and we don't spend hours downloading kernels for the kernel module test
# Abuse auto.conf since local.conf would be overwritten by the SDK
with open(os.path.join(sdk_dir, 'conf', 'auto.conf'), 'a+') as f:
f.write('SSTATE_MIRRORS += " \\n file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR'))
f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR'))
f.write('INHERIT += "own-mirrors"\n')
f.write('PREMIRRORS_prepend = " git://git.yoctoproject.org/.* git://%s/git2/git.yoctoproject.org.BASENAME \\n "\n' % test_data.get('DL_DIR'))
# We need to do this in case we have a minimal SDK
subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % \
sdk_env, cwd=sdk_dir, shell=True, stderr=subprocess.STDOUT)
tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir,
sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
host_pkg_manifest=host_pkg_manifest)
try:
tc.loadTests(OESDKExtTestContextExecutor.default_cases)
except Exception as e:
import traceback
bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
result = tc.runTests()
component = "%s %s" % (pn, OESDKExtTestContextExecutor.name)
context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
configuration = self.get_sdk_configuration(d, 'sdkext')
result.logDetails(self.get_sdk_json_result_dir(d),
configuration,
self.get_sdk_result_id(configuration))
result.logSummary(component, context_msg)
if not result.wasSuccessful():
fail = True
if fail:
bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
def run(self, d):
import os
import subprocess
import json
import logging
from bb.utils import export_proxies
from oeqa.utils import make_logger_bitbake_compatible
pn = d.getVar("PN")
logger = make_logger_bitbake_compatible(logging.getLogger("BitBake"))
# sdk use network for download projects for build
export_proxies(d)
tcname = self.get_tcname(d)
if not os.path.exists(tcname):
bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname)
tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json")
test_data = json.load(open(tdname, "r"))
target_pkg_manifest = self.context_executor_class._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest"))
host_pkg_manifest = self.context_executor_class._load_manifest(
d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest"))
processes = d.getVar("TESTIMAGE_NUMBER_THREADS") or d.getVar("BB_NUMBER_THREADS")
if processes:
try:
import testtools, subunit
except ImportError:
bb.warn("Failed to import testtools or subunit, the testcases will run serially")
processes = None
sdk_dir = d.expand("${WORKDIR}/testimage-sdk/")
bb.utils.remove(sdk_dir, True)
bb.utils.mkdirhier(sdk_dir)
context_args = self.setup_context(d)
self.extract_sdk(tcname, sdk_dir, d)
fail = False
sdk_envs = self.context_executor_class._get_sdk_environs(sdk_dir)
for s in sdk_envs:
sdk_env = sdk_envs[s]
bb.plain("SDK testing environment: %s" % s)
tc = self.context_class(td=test_data, logger=logger, sdk_dir=sdk_dir,
sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest,
host_pkg_manifest=host_pkg_manifest, **context_args)
try:
tc.loadTests(self.context_executor_class.default_cases)
except Exception as e:
import traceback
bb.fatal("Loading tests failed:\n%s" % traceback.format_exc())
if processes:
result = tc.runTests(processes=int(processes))
else:
result = tc.runTests()
component = "%s %s" % (pn, self.context_executor_class.name)
context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env))
configuration = self.get_sdk_configuration(d, self.test_type)
result.logDetails(self.get_sdk_json_result_dir(d),
configuration,
self.get_sdk_result_id(configuration))
result.logSummary(component, context_msg)
if not result.wasSuccessful():
fail = True
if fail:
bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
