def create_socket(url, d): import urllib socket = None try: export_proxies(d) socket = urllib.request.urlopen(url) except: bb.warn("distro_check: create_socket url %s can't access" % url) return socket
def create_socket(url, d): import urllib socket = None try: export_proxies(d) socket = urllib.request.urlopen(url) except: bb.warn("distro_check: create_socket url %s can't access" % url) return socket
def checkstatus(self, fetch, ud, d, try_again=True): import urllib.request, urllib.error, urllib.parse, socket, http.client from urllib.response import addinfourl from bb.fetch2 import FetchConnectionCache class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self): """Connect to the host and port specified in __init__.""" sock = fetch.connection_cache.get_connection(self.host, self.port) if sock: self.sock = sock else: self.sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address) fetch.connection_cache.add_connection(self.host, self.port, self.sock) if self._tunnel_host: self._tunnel() class CacheHTTPHandler(urllib.request.HTTPHandler): def http_open(self, req): return self.do_open(HTTPConnectionCache, req) def do_open(self, http_class, req): """Return an addinfourl object for the request, using http_class. http_class must implement the HTTPConnection API from httplib. The addinfourl return value is a file-like object. It also has methods and attributes including: - info(): return a mimetools.Message object for the headers - geturl(): return the original request URL - code: HTTP status code """ host = req.host if not host: raise urlllib2.URLError('no host given') h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) headers.update(dict((k, v) for k, v in list(req.headers.items()) if k not in headers)) # We want to make an HTTP/1.1 request, but the addinfourl # class isn't prepared to deal with a persistent connection. # It will try to read all remaining data from the socket, # which will block while the server waits for the next request. # So make sure the connection gets closed after the (only) # request. # Don't close connection when connection_cache is enabled, if fetch.connection_cache is None: headers["Connection"] = "close" else: headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 headers = dict( (name.title(), val) for name, val in list(headers.items())) if req._tunnel_host: tunnel_headers = {} proxy_auth_hdr = "Proxy-Authorization" if proxy_auth_hdr in headers: tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] # Proxy-Authorization should not be sent to origin # server. del headers[proxy_auth_hdr] h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: h.request(req.get_method(), req.selector, req.data, headers) except socket.error as err: # XXX what error? # Don't close connection when cache is enabled. # Instead, try to detect connections that are no longer # usable (for example, closed unexpectedly) and remove # them from the cache. if fetch.connection_cache is None: h.close() elif isinstance(err, OSError) and err.errno == errno.EBADF: # This happens when the server closes the connection despite the Keep-Alive. # Apparently urllib then uses the file descriptor, expecting it to be # connected, when in reality the connection is already gone. # We let the request fail and expect it to be # tried once more ("try_again" in check_status()), # with the dead connection removed from the cache. # If it still fails, we give up, which can happend for bad # HTTP proxy settings. fetch.connection_cache.remove_connection(h.host, h.port) raise urllib.error.URLError(err) else: try: r = h.getresponse(buffering=True) except TypeError: # buffering kw not supported r = h.getresponse() # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. # Wrap the HTTPResponse object in socket's file object adapter # for Windows. That adapter calls recv(), so delegate recv() # to read(). This weird wrapping allows the returned object to # have readline() and readlines() methods. # XXX It might be better to extract the read buffering code # out of socket._fileobject() and into a base class. r.recv = r.read # no data, just have to read r.read() class fp_dummy(object): def read(self): return "" def readline(self): return "" def close(self): pass resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason # Close connection when server request it. if fetch.connection_cache is not None: if 'Connection' in r.msg and r.msg['Connection'] == 'close': fetch.connection_cache.remove_connection(h.host, h.port) return resp class HTTPMethodFallback(urllib.request.BaseHandler): """ Fallback to GET if HEAD is not allowed (405 HTTP error) """ def http_error_405(self, req, fp, code, msg, headers): fp.read() fp.close() newheaders = dict((k,v) for k,v in list(req.headers.items()) if k.lower() not in ("content-length", "content-type")) return self.parent.open(urllib.request.Request(req.get_full_url(), headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True)) """ Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 Forbidden when they actually mean 405 Method Not Allowed. """ http_error_403 = http_error_405 class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): """ urllib2.HTTPRedirectHandler resets the method to GET on redirect, when we want to follow redirects using the original method. """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) newreq.get_method = lambda: req.get_method() return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] if export_proxies: handlers.append(urllib.request.ProxyHandler()) handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl if hasattr(ssl, '_create_unverified_context'): handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) opener = urllib.request.build_opener(*handlers) try: uri = ud.url.split(";")[0] r = urllib.request.Request(uri) r.get_method = lambda: "HEAD" # Some servers (FusionForge, as used on Alioth) require that the # optional Accept header is set. r.add_header("Accept", "*/*") def add_basic_auth(login_str, request): '''Adds Basic auth to http request, pass in login:password as string''' import base64 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") authheader = "Basic %s" % encodeuser r.add_header("Authorization", authheader) if ud.user: add_basic_auth(ud.user, r) try: import netrc, urllib.parse n = netrc.netrc() login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) add_basic_auth("%s:%s" % (login, password), r) except (TypeError, ImportError, IOError, netrc.NetrcParseError): pass opener.open(r) except urllib.error.URLError as e: if try_again: logger.debug(2, "checkstatus: trying again") return self.checkstatus(fetch, ud, d, False) else: # debug for now to avoid spamming the logs in e.g. remote sstate searches logger.debug(2, "checkstatus() urlopen failed: %s" % e) return False return True
def checkstatus(self, fetch, ud, d, try_again=True): import urllib.request, urllib.error, urllib.parse, socket, http.client from urllib.response import addinfourl from bb.fetch2 import FetchConnectionCache class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self): """Connect to the host and port specified in __init__.""" sock = fetch.connection_cache.get_connection(self.host, self.port) if sock: self.sock = sock else: self.sock = socket.create_connection((self.host, self.port), self.timeout, self.source_address) fetch.connection_cache.add_connection(self.host, self.port, self.sock) if self._tunnel_host: self._tunnel() class CacheHTTPHandler(urllib.request.HTTPHandler): def http_open(self, req): return self.do_open(HTTPConnectionCache, req) def do_open(self, http_class, req): """Return an addinfourl object for the request, using http_class. http_class must implement the HTTPConnection API from httplib. The addinfourl return value is a file-like object. It also has methods and attributes including: - info(): return a mimetools.Message object for the headers - geturl(): return the original request URL - code: HTTP status code """ host = req.host if not host: raise urlllib2.URLError('no host given') h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) headers.update(dict((k, v) for k, v in list(req.headers.items()) if k not in headers)) # We want to make an HTTP/1.1 request, but the addinfourl # class isn't prepared to deal with a persistent connection. # It will try to read all remaining data from the socket, # which will block while the server waits for the next request. # So make sure the connection gets closed after the (only) # request. # Don't close connection when connection_cache is enabled, if fetch.connection_cache is None: headers["Connection"] = "close" else: headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 headers = dict( (name.title(), val) for name, val in list(headers.items())) if req._tunnel_host: tunnel_headers = {} proxy_auth_hdr = "Proxy-Authorization" if proxy_auth_hdr in headers: tunnel_headers[proxy_auth_hdr] = headers[proxy_auth_hdr] # Proxy-Authorization should not be sent to origin # server. del headers[proxy_auth_hdr] h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: h.request(req.get_method(), req.selector, req.data, headers) except socket.error as err: # XXX what error? # Don't close connection when cache is enabled. if fetch.connection_cache is None: h.close() raise urllib.error.URLError(err) else: try: r = h.getresponse(buffering=True) except TypeError: # buffering kw not supported r = h.getresponse() # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. # Wrap the HTTPResponse object in socket's file object adapter # for Windows. That adapter calls recv(), so delegate recv() # to read(). This weird wrapping allows the returned object to # have readline() and readlines() methods. # XXX It might be better to extract the read buffering code # out of socket._fileobject() and into a base class. r.recv = r.read # no data, just have to read r.read() class fp_dummy(object): def read(self): return "" def readline(self): return "" def close(self): pass resp = addinfourl(fp_dummy(), r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason # Close connection when server request it. if fetch.connection_cache is not None: if 'Connection' in r.msg and r.msg['Connection'] == 'close': fetch.connection_cache.remove_connection(h.host, h.port) return resp class HTTPMethodFallback(urllib.request.BaseHandler): """ Fallback to GET if HEAD is not allowed (405 HTTP error) """ def http_error_405(self, req, fp, code, msg, headers): fp.read() fp.close() newheaders = dict((k,v) for k,v in list(req.headers.items()) if k.lower() not in ("content-length", "content-type")) return self.parent.open(urllib.request.Request(req.get_full_url(), headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True)) """ Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 Forbidden when they actually mean 405 Method Not Allowed. """ http_error_403 = http_error_405 """ Some servers (e.g. FusionForge) returns 406 Not Acceptable when they actually mean 405 Method Not Allowed. """ http_error_406 = http_error_405 class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): """ urllib2.HTTPRedirectHandler resets the method to GET on redirect, when we want to follow redirects using the original method. """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib.request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) newreq.get_method = lambda: req.get_method() return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] if export_proxies: handlers.append(urllib.request.ProxyHandler()) handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl if hasattr(ssl, '_create_unverified_context'): handlers.append(urllib.request.HTTPSHandler(context=ssl._create_unverified_context())) opener = urllib.request.build_opener(*handlers) try: uri = ud.url.split(";")[0] r = urllib.request.Request(uri) r.get_method = lambda: "HEAD" def add_basic_auth(login_str, request): '''Adds Basic auth to http request, pass in login:password as string''' import base64 encodeuser = base64.b64encode(login_str.encode('utf-8')).decode("utf-8") authheader = "Basic %s" % encodeuser r.add_header("Authorization", authheader) if ud.user: add_basic_auth(ud.user, r) try: import netrc, urllib.parse n = netrc.netrc() login, unused, password = n.authenticators(urllib.parse.urlparse(uri).hostname) add_basic_auth("%s:%s" % (login, password), r) except (TypeError, ImportError, IOError, netrc.NetrcParseError): pass opener.open(r) except urllib.error.URLError as e: if try_again: logger.debug(2, "checkstatus: trying again") return self.checkstatus(fetch, ud, d, False) else: # debug for now to avoid spamming the logs in e.g. remote sstate searches logger.debug(2, "checkstatus() urlopen failed: %s" % e) return False return True
def download_old_versions(d): """ Download the necessary information from the update repo that is needed to build updates in that update stream. This can run in parallel to a normal build and thus is not on the critical path. """ content_url = d.getVar('SWUPD_CONTENT_BUILD_URL', True) version_url = d.getVar('SWUPD_VERSION_BUILD_URL', True) current_format = int(d.getVar('SWUPD_FORMAT', True)) deploy_dir = d.getVar('DEPLOY_DIR_SWUPD', True) www_dir = os.path.join(deploy_dir, 'www') if not content_url or not version_url: bb.warn( 'SWUPD_CONTENT_BUILD_URL and/or SWUPD_VERSION_BUILD_URL not set, skipping download of old versions for the initial build of a swupd update stream.' ) return # Avoid double // in path. At least twisted is sensitive to that. content_url = content_url.rstrip('/') # Set up env variables with proxy information for use in urllib. export_proxies(d) # Find latest version for each of the older formats. # For now we ignore the released milestones and go # directly to the URL with all builds. The information # about milestones may be relevant for determining # how format changes need to be handled. latest_versions = {} for format in range(3, current_format + 1): try: url = '%s/version/format%d/latest' % (content_url, format) response = urllib.request.urlopen(url) version = int(response.read()) latest_versions[format] = version formatdir = os.path.join(www_dir, 'version', 'format%d' % format) bb.utils.mkdirhier(formatdir) with open(os.path.join(formatdir, 'latest'), 'w') as latest: latest.write(str(version)) except urllib.error.HTTPError as http_error: if http_error.code == 404: bb.debug(1, '%s does not exist, skipping that format' % url) else: raise except urllib.error.URLError as url_error: # Happens for file:// URLs. if isinstance(url_error.reason, OSError) and url_error.reason.errno == errno.ENOENT: bb.debug(1, '%s does not exist, skipping that format' % url) else: raise # Now get the Manifests of the latest versions and the # versions we are supposed to provide a delta for, as a starting point. # In addition, we also need Manifests that provide files reused by # these initial set of Manifests or get referenced by them. # # There's no integrity checking for the files. bsdtar is # expected to detect corrupted archives and https is expected # to protect against man-in-the-middle attacks. pending_versions = set(latest_versions.values()) pending_versions.update( [int(x) for x in d.getVar('SWUPD_DELTAPACK_VERSIONS', True).split()]) fetched_versions = set([0]) while pending_versions: version = pending_versions.pop() sub_versions = set() sub_versions.update( download_manifests(content_url, version, 'MoM', os.path.join(www_dir, str(version)))) sub_versions.update( download_manifests(content_url, version, 'full', os.path.join(www_dir, str(version)))) fetched_versions.add(version) pending_versions.update(sub_versions.difference(fetched_versions)) latest_version_file = os.path.join(deploy_dir, 'image', 'latest.version') if not os.path.exists(latest_version_file): # We located information about latest version from online www update repo. # Now use that to determine what we are updating from. Doing this here # instead of swupd-image.bbclass has the advantage that we can do some # sanity checking very early in a build. # # Building a proper update makes swupd_create_fullfiles # a lot faster because it allows reusing existing, unmodified files. # Saves a lot of space, too, because the new Manifest files then merely # point to the older version (no entry in ${DEPLOY_DIR_SWUPD}/www/${OS_VERSION}/files, # not even a link). if not latest_versions: bb.fatal( "%s does not exist and no information was found under SWUPD_CONTENT_BUILD_URL %s, cannot proceed without information about the previous build. When building the initial version, unset SWUPD_VERSION_BUILD_URL and SWUPD_CONTENT_BUILD_URL to proceed." % (latest_version_file, content_url)) latest = sorted(latest_versions.values())[-1] bb.debug(2, "Setting %d in latest.version file" % latest) with open(latest_version_file, 'w') as f: f.write(str(latest))
actually mean 405 Method Not Allowed. """ http_error_406 = http_error_405 class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): """ urllib2.HTTPRedirectHandler resets the method to GET on redirect, when we want to follow redirects using the original method. """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib2.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, newurl) newreq.get_method = lambda: req.get_method() return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] if export_proxies: handlers.append(urllib2.ProxyHandler()) handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl if hasattr(ssl, "_create_unverified_context"): handlers.append(urllib2.HTTPSHandler(context=ssl._create_unverified_context())) opener = urllib2.build_opener(*handlers) try:
def run(self, d): import os import subprocess import json import logging from bb.utils import export_proxies from oeqa.utils import make_logger_bitbake_compatible pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) # sdk use network for download projects for build export_proxies(d) tcname = self.get_tcname(d) if not os.path.exists(tcname): bb.fatal( "The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname) tdname = d.expand( "${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json") test_data = json.load(open(tdname, "r")) target_pkg_manifest = self.context_executor_class._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest")) host_pkg_manifest = self.context_executor_class._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest")) processes = d.getVar("TESTIMAGE_NUMBER_THREADS") or d.getVar( "BB_NUMBER_THREADS") if processes: try: import testtools, subunit except ImportError: bb.warn( "Failed to import testtools or subunit, the testcases will run serially" ) processes = None sdk_dir = d.expand("${WORKDIR}/testimage-sdk/") bb.utils.remove(sdk_dir, True) bb.utils.mkdirhier(sdk_dir) context_args = self.setup_context(d) self.extract_sdk(tcname, sdk_dir, d) fail = False sdk_envs = self.context_executor_class._get_sdk_environs(sdk_dir) for s in sdk_envs: sdk_env = sdk_envs[s] bb.plain("SDK testing environment: %s" % s) tc = self.context_class(td=test_data, logger=logger, sdk_dir=sdk_dir, sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest, host_pkg_manifest=host_pkg_manifest, **context_args) try: tc.loadTests(self.context_executor_class.default_cases) except Exception as e: import traceback bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) if processes: result = tc.runTests(processes=int(processes)) else: result = tc.runTests() component = "%s %s" % (pn, self.context_executor_class.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) configuration = self.get_sdk_configuration(d, self.test_type) result.logDetails(self.get_sdk_json_result_dir(d), configuration, self.get_sdk_result_id(configuration)) result.logSummary(component, context_msg) if not result.wasSuccessful(): fail = True if fail: bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
def create_socket(url, d): import urllib from bb.utils import export_proxies export_proxies(d) return urllib.request.urlopen(url)
actually mean 405 Method Not Allowed. """ http_error_406 = http_error_405 class FixedHTTPRedirectHandler(urllib2.HTTPRedirectHandler): """ urllib2.HTTPRedirectHandler resets the method to GET on redirect, when we want to follow redirects using the original method. """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib2.HTTPRedirectHandler.redirect_request( self, req, fp, code, msg, headers, newurl) newreq.get_method = lambda: req.get_method() return newreq exported_proxies = export_proxies(d) handlers = [FixedHTTPRedirectHandler, HTTPMethodFallback] if export_proxies: handlers.append(urllib2.ProxyHandler()) handlers.append(CacheHTTPHandler()) # XXX: Since Python 2.7.9 ssl cert validation is enabled by default # see PEP-0476, this causes verification errors on some https servers # so disable by default. import ssl if hasattr(ssl, '_create_unverified_context'): handlers.append( urllib2.HTTPSHandler(context=ssl._create_unverified_context())) opener = urllib2.build_opener(*handlers) try:
def create_socket(url, d): import urllib from bb.utils import export_proxies export_proxies(d) return urllib.request.urlopen(url)
def run(self, d): import os import json import subprocess import logging from bb.utils import export_proxies from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) # extensible sdk use network export_proxies(d) subprocesstweak.errors_have_output() # We need the original PATH for testing the eSDK, not with our manipulations os.environ['PATH'] = d.getVar("BB_ORIGENV", False).getVar("PATH") tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh") if not os.path.exists(tcname): bb.fatal("The toolchain ext %s is not built. Build it before running the" \ " tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname) tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json") test_data = json.load(open(tdname, "r")) target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest")) host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest")) sdk_dir = d.expand("${WORKDIR}/testsdkext/") bb.utils.remove(sdk_dir, True) bb.utils.mkdirhier(sdk_dir) try: subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True) except subprocess.CalledProcessError as e: msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8") logfn = os.path.join(sdk_dir, 'preparing_build_system.log') if os.path.exists(logfn): msg += '\n\nContents of preparing_build_system.log:\n' with open(logfn, 'r') as f: for line in f: msg += line bb.fatal(msg) fail = False sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir) for s in sdk_envs: bb.plain("Extensible SDK testing environment: %s" % s) sdk_env = sdk_envs[s] # Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache # and we don't spend hours downloading kernels for the kernel module test # Abuse auto.conf since local.conf would be overwritten by the SDK with open(os.path.join(sdk_dir, 'conf', 'auto.conf'), 'a+') as f: f.write('SSTATE_MIRRORS += " \\n file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR')) f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR')) f.write('INHERIT += "own-mirrors"\n') f.write('PREMIRRORS:prepend = " git://git.yoctoproject.org/.* git://%s/git2/git.yoctoproject.org.BASENAME \\n "\n' % test_data.get('DL_DIR')) # We need to do this in case we have a minimal SDK subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % \ sdk_env, cwd=sdk_dir, shell=True, stderr=subprocess.STDOUT) tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir, sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest, host_pkg_manifest=host_pkg_manifest) try: tc.loadTests(OESDKExtTestContextExecutor.default_cases) except Exception as e: import traceback bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) result = tc.runTests() component = "%s %s" % (pn, OESDKExtTestContextExecutor.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) configuration = self.get_sdk_configuration(d, 'sdkext') result.logDetails(self.get_sdk_json_result_dir(d), configuration, self.get_sdk_result_id(configuration)) result.logSummary(component, context_msg) if not result.wasSuccessful(): fail = True # Clean the workspace/sources to avoid `devtool add' failure because of non-empty source directory bb.utils.remove(sdk_dir+'workspace/sources', True) if fail: bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
def run(self, d): import os import json import subprocess import logging from bb.utils import export_proxies from oeqa.utils import avoid_paths_in_environ, make_logger_bitbake_compatible, subprocesstweak from oeqa.sdkext.context import OESDKExtTestContext, OESDKExtTestContextExecutor pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) # extensible sdk use network export_proxies(d) subprocesstweak.errors_have_output() # extensible sdk can be contaminated if native programs are # in PATH, i.e. use perl-native instead of eSDK one. paths_to_avoid = [d.getVar('STAGING_DIR'), d.getVar('BASE_WORKDIR')] os.environ['PATH'] = avoid_paths_in_environ(paths_to_avoid) tcname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.sh") if not os.path.exists(tcname): bb.fatal("The toolchain ext %s is not built. Build it before running the" \ " tests: 'bitbake <image> -c populate_sdk_ext' ." % tcname) tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.testdata.json") test_data = json.load(open(tdname, "r")) target_pkg_manifest = OESDKExtTestContextExecutor._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.target.manifest")) host_pkg_manifest = OESDKExtTestContextExecutor._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAINEXT_OUTPUTNAME}.host.manifest")) sdk_dir = d.expand("${WORKDIR}/testsdkext/") bb.utils.remove(sdk_dir, True) bb.utils.mkdirhier(sdk_dir) try: subprocess.check_output("%s -y -d %s" % (tcname, sdk_dir), shell=True) except subprocess.CalledProcessError as e: msg = "Couldn't install the extensible SDK:\n%s" % e.output.decode("utf-8") logfn = os.path.join(sdk_dir, 'preparing_build_system.log') if os.path.exists(logfn): msg += '\n\nContents of preparing_build_system.log:\n' with open(logfn, 'r') as f: for line in f: msg += line bb.fatal(msg) fail = False sdk_envs = OESDKExtTestContextExecutor._get_sdk_environs(sdk_dir) for s in sdk_envs: bb.plain("Extensible SDK testing environment: %s" % s) sdk_env = sdk_envs[s] # Use our own SSTATE_DIR and DL_DIR so that updates to the eSDK come from our sstate cache # and we don't spend hours downloading kernels for the kernel module test # Abuse auto.conf since local.conf would be overwritten by the SDK with open(os.path.join(sdk_dir, 'conf', 'auto.conf'), 'a+') as f: f.write('SSTATE_MIRRORS += " \\n file://.* file://%s/PATH"\n' % test_data.get('SSTATE_DIR')) f.write('SOURCE_MIRROR_URL = "file://%s"\n' % test_data.get('DL_DIR')) f.write('INHERIT += "own-mirrors"\n') f.write('PREMIRRORS_prepend = " git://git.yoctoproject.org/.* git://%s/git2/git.yoctoproject.org.BASENAME \\n "\n' % test_data.get('DL_DIR')) # We need to do this in case we have a minimal SDK subprocess.check_output(". %s > /dev/null; devtool sdk-install meta-extsdk-toolchain" % \ sdk_env, cwd=sdk_dir, shell=True, stderr=subprocess.STDOUT) tc = OESDKExtTestContext(td=test_data, logger=logger, sdk_dir=sdk_dir, sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest, host_pkg_manifest=host_pkg_manifest) try: tc.loadTests(OESDKExtTestContextExecutor.default_cases) except Exception as e: import traceback bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) result = tc.runTests() component = "%s %s" % (pn, OESDKExtTestContextExecutor.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) configuration = self.get_sdk_configuration(d, 'sdkext') result.logDetails(self.get_sdk_json_result_dir(d), configuration, self.get_sdk_result_id(configuration)) result.logSummary(component, context_msg) if not result.wasSuccessful(): fail = True if fail: bb.fatal("%s - FAILED - check the task log and the commands log" % pn)
def run(self, d): import os import subprocess import json import logging from bb.utils import export_proxies from oeqa.utils import make_logger_bitbake_compatible pn = d.getVar("PN") logger = make_logger_bitbake_compatible(logging.getLogger("BitBake")) # sdk use network for download projects for build export_proxies(d) tcname = self.get_tcname(d) if not os.path.exists(tcname): bb.fatal("The toolchain %s is not built. Build it before running the tests: 'bitbake <image> -c populate_sdk' ." % tcname) tdname = d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.testdata.json") test_data = json.load(open(tdname, "r")) target_pkg_manifest = self.context_executor_class._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.target.manifest")) host_pkg_manifest = self.context_executor_class._load_manifest( d.expand("${SDK_DEPLOY}/${TOOLCHAIN_OUTPUTNAME}.host.manifest")) processes = d.getVar("TESTIMAGE_NUMBER_THREADS") or d.getVar("BB_NUMBER_THREADS") if processes: try: import testtools, subunit except ImportError: bb.warn("Failed to import testtools or subunit, the testcases will run serially") processes = None sdk_dir = d.expand("${WORKDIR}/testimage-sdk/") bb.utils.remove(sdk_dir, True) bb.utils.mkdirhier(sdk_dir) context_args = self.setup_context(d) self.extract_sdk(tcname, sdk_dir, d) fail = False sdk_envs = self.context_executor_class._get_sdk_environs(sdk_dir) for s in sdk_envs: sdk_env = sdk_envs[s] bb.plain("SDK testing environment: %s" % s) tc = self.context_class(td=test_data, logger=logger, sdk_dir=sdk_dir, sdk_env=sdk_env, target_pkg_manifest=target_pkg_manifest, host_pkg_manifest=host_pkg_manifest, **context_args) try: tc.loadTests(self.context_executor_class.default_cases) except Exception as e: import traceback bb.fatal("Loading tests failed:\n%s" % traceback.format_exc()) if processes: result = tc.runTests(processes=int(processes)) else: result = tc.runTests() component = "%s %s" % (pn, self.context_executor_class.name) context_msg = "%s:%s" % (os.path.basename(tcname), os.path.basename(sdk_env)) configuration = self.get_sdk_configuration(d, self.test_type) result.logDetails(self.get_sdk_json_result_dir(d), configuration, self.get_sdk_result_id(configuration)) result.logSummary(component, context_msg) if not result.wasSuccessful(): fail = True if fail: bb.fatal("%s - FAILED - check the task log and the commands log" % pn)