def _runwget(self, ud, d, command, quiet, workdir=None): progresshandler = WgetProgressHandler(d) logger.debug2("Fetching %s using command '%s'" % (ud.url, command)) bb.fetch2.check_network_access(d, command, ud.url) runfetchcmd(command + ' --progress=dot -v', d, quiet, log=progresshandler, workdir=workdir)
def download(self, ud, d, retries=3): """Fetch urls""" # If were reaching the account transaction limit we might be refused a connection, # retrying allows us to avoid false negatives since the limit changes over time fetchcmd = self.basecmd + ' --retry-connrefused --waitretry=5' # We need to provide a localpath to avoid wget using the SAS # ud.localfile either has the downloadfilename or ud.path localpath = os.path.join(d.getVar("DL_DIR"), ud.localfile) bb.utils.mkdirhier(os.path.dirname(localpath)) fetchcmd += " -O %s" % shlex.quote(localpath) if ud.user and ud.pswd: fetchcmd += " --user=%s --password=%s --auth-no-challenge" % ( ud.user, ud.pswd) # Check if a Shared Access Signature was given and use it az_sas = d.getVar('AZ_SAS') if az_sas: azuri = '%s%s%s%s' % ('https://', ud.host, ud.path, az_sas) else: azuri = '%s%s%s' % ('https://', ud.host, ud.path) if os.path.exists(ud.localpath): # file exists, but we didnt complete it.. trying again. fetchcmd += d.expand(" -c -P ${DL_DIR} '%s'" % azuri) else: fetchcmd += d.expand(" -P ${DL_DIR} '%s'" % azuri) try: self._runwget(ud, d, fetchcmd, False) except FetchError as e: # Azure fails on handshake sometimes when using wget after some stress, producing a # FetchError from the fetcher, if the artifact exists retyring should succeed if 'Unable to establish SSL connection' in str(e): logger.debug2( 'Unable to establish SSL connection: Retries remaining: %s, Retrying...' % retries) self.download(ud, d, retries - 1) # Sanity check since wget can pretend it succeed when it didn't # Also, this used to happen if sourceforge sent us to the mirror page if not os.path.exists(ud.localpath): raise FetchError( "The fetch command returned success for url %s but %s doesn't exist?!" % (azuri, ud.localpath), azuri) if os.path.getsize(ud.localpath) == 0: os.remove(ud.localpath) raise FetchError( "The fetch of %s resulted in a zero size file?! Deleting and failing since this isn't right." % (azuri), azuri) return True
def download(self, ud, d): """Fetch url""" logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") lf = bb.utils.lockfile(ud.svnlock) try: if os.access(os.path.join(ud.moddir, '.svn'), os.R_OK): svncmd = self._buildsvncommand(ud, d, "update") logger.info("Update " + ud.url) # We need to attempt to run svn upgrade first in case its an older working format try: runfetchcmd(ud.basecmd + " upgrade", d, workdir=ud.moddir) except FetchError: pass logger.debug("Running %s", svncmd) bb.fetch2.check_network_access(d, svncmd, ud.url) runfetchcmd(svncmd, d, workdir=ud.moddir) else: svncmd = self._buildsvncommand(ud, d, "fetch") logger.info("Fetch " + ud.url) # check out sources there bb.utils.mkdirhier(ud.pkgdir) logger.debug("Running %s", svncmd) bb.fetch2.check_network_access(d, svncmd, ud.url) runfetchcmd(svncmd, d, workdir=ud.pkgdir) if not ("externals" in ud.parm and ud.parm["externals"] == "nowarn"): # Warn the user if this had externals (won't catch them all) output = runfetchcmd("svn propget svn:externals || true", d, workdir=ud.moddir) if output: if "--ignore-externals" in svncmd.split(): bb.warn("%s contains svn:externals." % ud.url) bb.warn("These should be added to the recipe SRC_URI as necessary.") bb.warn("svn fetch has ignored externals:\n%s" % output) bb.warn("To disable this warning add ';externals=nowarn' to the url.") else: bb.debug(1, "svn repository has externals:\n%s" % output) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude='.svn'" # tar them up to a defined filename runfetchcmd("tar %s -czf %s %s" % (tar_flags, ud.localpath, ud.path_spec), d, cleanup=[ud.localpath], workdir=ud.pkgdir) finally: bb.utils.unlockfile(lf)
def localpaths(self, urldata, d): """ Return the local filename of a given url assuming a successful fetch. """ searched = [] path = urldata.decodedurl newpath = path if path[0] == "/": return [path] filespath = d.getVar('FILESPATH') if filespath: logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) newpath, hist = bb.utils.which(filespath, path, history=True) searched.extend(hist) return searched
def download(self, ud, d): """Fetch url""" logger.debug2("Fetch: checking for module directory '" + ud.moddir + "'") # If the checkout doesn't exist and the mirror tarball does, extract it if not os.path.exists(ud.pkgdir) and os.path.exists(ud.fullmirror): bb.utils.mkdirhier(ud.pkgdir) runfetchcmd("tar -xzf %s" % (ud.fullmirror), d, workdir=ud.pkgdir) if os.access(os.path.join(ud.moddir, '.hg'), os.R_OK): # Found the source, check whether need pull updatecmd = self._buildhgcommand(ud, d, "update") logger.debug("Running %s", updatecmd) try: runfetchcmd(updatecmd, d, workdir=ud.moddir) except bb.fetch2.FetchError: # Runnning pull in the repo pullcmd = self._buildhgcommand(ud, d, "pull") logger.info("Pulling " + ud.url) # update sources there logger.debug("Running %s", pullcmd) bb.fetch2.check_network_access(d, pullcmd, ud.url) runfetchcmd(pullcmd, d, workdir=ud.moddir) try: os.unlink(ud.fullmirror) except OSError as exc: if exc.errno != errno.ENOENT: raise # No source found, clone it. if not os.path.exists(ud.moddir): fetchcmd = self._buildhgcommand(ud, d, "fetch") logger.info("Fetch " + ud.url) # check out sources there bb.utils.mkdirhier(ud.pkgdir) logger.debug("Running %s", fetchcmd) bb.fetch2.check_network_access(d, fetchcmd, ud.url) runfetchcmd(fetchcmd, d, workdir=ud.pkgdir) # Even when we clone (fetch), we still need to update as hg's clone # won't checkout the specified revision if its on a branch updatecmd = self._buildhgcommand(ud, d, "update") logger.debug("Running %s", updatecmd) runfetchcmd(updatecmd, d, workdir=ud.moddir)
def localpaths(self, urldata, d): """ Return the local filename of a given url assuming a successful fetch. """ searched = [] path = urldata.decodedurl newpath = path if path[0] == "/": return [path] filespath = d.getVar('FILESPATH') if filespath: logger.debug2("Searching for %s in paths:\n %s" % (path, "\n ".join(filespath.split(":")))) newpath, hist = bb.utils.which(filespath, path, history=True) searched.extend(hist) if not os.path.exists(newpath): dldirfile = os.path.join(d.getVar("DL_DIR"), path) logger.debug2("Defaulting to %s for %s" % (dldirfile, path)) bb.utils.mkdirhier(os.path.dirname(dldirfile)) searched.append(dldirfile) return searched return searched
def unpack(self, ud, destdir, d): """ Make a local clone or export for the url """ revflag = "-r %s" % ud.revision subdir = ud.parm.get("destsuffix", ud.module) codir = "%s/%s" % (destdir, subdir) scmdata = ud.parm.get("scmdata", "") if scmdata != "nokeep": proto = ud.parm.get('protocol', 'http') if not os.access(os.path.join(codir, '.hg'), os.R_OK): logger.debug2("Unpack: creating new hg repository in '" + codir + "'") runfetchcmd("%s init %s" % (ud.basecmd, codir), d) logger.debug2("Unpack: updating source in '" + codir + "'") if ud.user and ud.pswd: runfetchcmd( "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" pull %s" % (ud.basecmd, ud.user, ud.pswd, proto, ud.moddir), d, workdir=codir) else: runfetchcmd("%s pull %s" % (ud.basecmd, ud.moddir), d, workdir=codir) if ud.user and ud.pswd: runfetchcmd( "%s --config auth.default.prefix=* --config auth.default.username=%s --config auth.default.password=%s --config \"auth.default.schemes=%s\" up -C %s" % (ud.basecmd, ud.user, ud.pswd, proto, revflag), d, workdir=codir) else: runfetchcmd("%s up -C %s" % (ud.basecmd, revflag), d, workdir=codir) else: logger.debug2("Unpack: extracting source to '" + codir + "'") runfetchcmd("%s archive -t files %s %s" % (ud.basecmd, revflag, codir), d, workdir=ud.moddir)
def checkstatus(self, fetch, ud, d, try_again=True): class HTTPConnectionCache(http.client.HTTPConnection): if fetch.connection_cache: def connect(self): """Connect to the host and port specified in __init__.""" sock = fetch.connection_cache.get_connection( self.host, self.port) if sock: self.sock = sock else: self.sock = socket.create_connection( (self.host, self.port), self.timeout, self.source_address) fetch.connection_cache.add_connection( self.host, self.port, self.sock) if self._tunnel_host: self._tunnel() class CacheHTTPHandler(urllib.request.HTTPHandler): def http_open(self, req): return self.do_open(HTTPConnectionCache, req) def do_open(self, http_class, req): """Return an addinfourl object for the request, using http_class. http_class must implement the HTTPConnection API from httplib. The addinfourl return value is a file-like object. It also has methods and attributes including: - info(): return a mimetools.Message object for the headers - geturl(): return the original request URL - code: HTTP status code """ host = req.host if not host: raise urllib.error.URLError('no host given') h = http_class(host, timeout=req.timeout) # will parse host:port h.set_debuglevel(self._debuglevel) headers = dict(req.unredirected_hdrs) headers.update( dict((k, v) for k, v in list(req.headers.items()) if k not in headers)) # We want to make an HTTP/1.1 request, but the addinfourl # class isn't prepared to deal with a persistent connection. # It will try to read all remaining data from the socket, # which will block while the server waits for the next request. # So make sure the connection gets closed after the (only) # request. # Don't close connection when connection_cache is enabled, if fetch.connection_cache is None: headers["Connection"] = "close" else: headers["Connection"] = "Keep-Alive" # Works for HTTP/1.0 headers = dict( (name.title(), val) for name, val in list(headers.items())) if req._tunnel_host: tunnel_headers = {} proxy_auth_hdr = "Proxy-Authorization" if proxy_auth_hdr in headers: tunnel_headers[proxy_auth_hdr] = headers[ proxy_auth_hdr] # Proxy-Authorization should not be sent to origin # server. del headers[proxy_auth_hdr] h.set_tunnel(req._tunnel_host, headers=tunnel_headers) try: h.request(req.get_method(), req.selector, req.data, headers) except socket.error as err: # XXX what error? # Don't close connection when cache is enabled. # Instead, try to detect connections that are no longer # usable (for example, closed unexpectedly) and remove # them from the cache. if fetch.connection_cache is None: h.close() elif isinstance(err, OSError) and err.errno == errno.EBADF: # This happens when the server closes the connection despite the Keep-Alive. # Apparently urllib then uses the file descriptor, expecting it to be # connected, when in reality the connection is already gone. # We let the request fail and expect it to be # tried once more ("try_again" in check_status()), # with the dead connection removed from the cache. # If it still fails, we give up, which can happend for bad # HTTP proxy settings. fetch.connection_cache.remove_connection( h.host, h.port) raise urllib.error.URLError(err) else: r = h.getresponse() # Pick apart the HTTPResponse object to get the addinfourl # object initialized properly. # Wrap the HTTPResponse object in socket's file object adapter # for Windows. That adapter calls recv(), so delegate recv() # to read(). This weird wrapping allows the returned object to # have readline() and readlines() methods. # XXX It might be better to extract the read buffering code # out of socket._fileobject() and into a base class. r.recv = r.read # no data, just have to read r.read() class fp_dummy(object): def read(self): return "" def readline(self): return "" def close(self): pass closed = False resp = urllib.response.addinfourl(fp_dummy(), r.msg, req.get_full_url()) resp.code = r.status resp.msg = r.reason # Close connection when server request it. if fetch.connection_cache is not None: if 'Connection' in r.msg and r.msg['Connection'] == 'close': fetch.connection_cache.remove_connection( h.host, h.port) return resp class HTTPMethodFallback(urllib.request.BaseHandler): """ Fallback to GET if HEAD is not allowed (405 HTTP error) """ def http_error_405(self, req, fp, code, msg, headers): fp.read() fp.close() if req.get_method() != 'GET': newheaders = dict( (k, v) for k, v in list(req.headers.items()) if k.lower() not in ("content-length", "content-type")) return self.parent.open( urllib.request.Request( req.get_full_url(), headers=newheaders, origin_req_host=req.origin_req_host, unverifiable=True)) raise urllib.request.HTTPError(req, code, msg, headers, None) # Some servers (e.g. GitHub archives, hosted on Amazon S3) return 403 # Forbidden when they actually mean 405 Method Not Allowed. http_error_403 = http_error_405 class FixedHTTPRedirectHandler(urllib.request.HTTPRedirectHandler): """ urllib2.HTTPRedirectHandler resets the method to GET on redirect, when we want to follow redirects using the original method. """ def redirect_request(self, req, fp, code, msg, headers, newurl): newreq = urllib.request.HTTPRedirectHandler.redirect_request( self, req, fp, code, msg, headers, newurl) newreq.get_method = req.get_method return newreq # We need to update the environment here as both the proxy and HTTPS # handlers need variables set. The proxy needs http_proxy and friends to # be set, and HTTPSHandler ends up calling into openssl to load the # certificates. In buildtools configurations this will be looking at the # wrong place for certificates by default: we set SSL_CERT_FILE to the # right location in the buildtools environment script but as BitBake # prunes prunes the environment this is lost. When binaries are executed # runfetchcmd ensures these values are in the environment, but this is # pure Python so we need to update the environment. # # Avoid tramping the environment too much by using bb.utils.environment # to scope the changes to the build_opener request, which is when the # environment lookups happen. newenv = {} for name in bb.fetch2.FETCH_EXPORT_VARS: value = d.getVar(name) if not value: origenv = d.getVar("BB_ORIGENV") if origenv: value = origenv.getVar(name) if value: newenv[name] = value with bb.utils.environment(**newenv): import ssl if self.check_certs(d): context = ssl.create_default_context() else: context = ssl._create_unverified_context() handlers = [ FixedHTTPRedirectHandler, HTTPMethodFallback, urllib.request.ProxyHandler(), CacheHTTPHandler(), urllib.request.HTTPSHandler(context=context) ] opener = urllib.request.build_opener(*handlers) try: uri = ud.url.split(";")[0] r = urllib.request.Request(uri) r.get_method = lambda: "HEAD" # Some servers (FusionForge, as used on Alioth) require that the # optional Accept header is set. r.add_header("Accept", "*/*") r.add_header("User-Agent", self.user_agent) def add_basic_auth(login_str, request): '''Adds Basic auth to http request, pass in login:password as string''' import base64 encodeuser = base64.b64encode( login_str.encode('utf-8')).decode("utf-8") authheader = "Basic %s" % encodeuser r.add_header("Authorization", authheader) if ud.user and ud.pswd: add_basic_auth(ud.user + ':' + ud.pswd, r) try: import netrc n = netrc.netrc() login, unused, password = n.authenticators( urllib.parse.urlparse(uri).hostname) add_basic_auth("%s:%s" % (login, password), r) except (TypeError, ImportError, IOError, netrc.NetrcParseError): pass with opener.open(r) as response: pass except urllib.error.URLError as e: if try_again: logger.debug2("checkstatus: trying again") return self.checkstatus(fetch, ud, d, False) else: # debug for now to avoid spamming the logs in e.g. remote sstate searches logger.debug2("checkstatus() urlopen failed: %s" % e) return False except ConnectionResetError as e: if try_again: logger.debug2("checkstatus: trying again") return self.checkstatus(fetch, ud, d, False) else: # debug for now to avoid spamming the logs in e.g. remote sstate searches logger.debug2("checkstatus() urlopen failed: %s" % e) return False return True
def download(self, ud, d): method = ud.parm.get('method', 'pserver') localdir = ud.parm.get('localdir', ud.module) cvs_port = ud.parm.get('port', '') cvs_rsh = None if method == "ext": if "rsh" in ud.parm: cvs_rsh = ud.parm["rsh"] if method == "dir": cvsroot = ud.path else: cvsroot = ":" + method cvsproxyhost = d.getVar('CVS_PROXY_HOST') if cvsproxyhost: cvsroot += ";proxy=" + cvsproxyhost cvsproxyport = d.getVar('CVS_PROXY_PORT') if cvsproxyport: cvsroot += ";proxyport=" + cvsproxyport cvsroot += ":" + ud.user if ud.pswd: cvsroot += ":" + ud.pswd cvsroot += "@" + ud.host + ":" + cvs_port + ud.path options = [] if 'norecurse' in ud.parm: options.append("-l") if ud.date: # treat YYYYMMDDHHMM specially for CVS if len(ud.date) == 12: options.append("-D \"%s %s:%s UTC\"" % (ud.date[0:8], ud.date[8:10], ud.date[10:12])) else: options.append("-D \"%s UTC\"" % ud.date) if ud.tag: options.append("-r %s" % ud.tag) cvsbasecmd = d.getVar("FETCHCMD_cvs") or "/usr/bin/env cvs" cvscmd = cvsbasecmd + " '-d" + cvsroot + "' co " + " ".join(options) + " " + ud.module cvsupdatecmd = cvsbasecmd + " '-d" + cvsroot + "' update -d -P " + " ".join(options) if cvs_rsh: cvscmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvscmd) cvsupdatecmd = "CVS_RSH=\"%s\" %s" % (cvs_rsh, cvsupdatecmd) # create module directory logger.debug2("Fetch: checking for module directory") moddir = os.path.join(ud.pkgdir, localdir) workdir = None if os.access(os.path.join(moddir, 'CVS'), os.R_OK): logger.info("Update " + ud.url) bb.fetch2.check_network_access(d, cvsupdatecmd, ud.url) # update sources there workdir = moddir cmd = cvsupdatecmd else: logger.info("Fetch " + ud.url) # check out sources there bb.utils.mkdirhier(ud.pkgdir) workdir = ud.pkgdir logger.debug("Running %s", cvscmd) bb.fetch2.check_network_access(d, cvscmd, ud.url) cmd = cvscmd runfetchcmd(cmd, d, cleanup=[moddir], workdir=workdir) if not os.access(moddir, os.R_OK): raise FetchError("Directory %s was not readable despite sucessful fetch?!" % moddir, ud.url) scmdata = ud.parm.get("scmdata", "") if scmdata == "keep": tar_flags = "" else: tar_flags = "--exclude='CVS'" # tar them up to a defined filename workdir = None if 'fullpath' in ud.parm: workdir = ud.pkgdir cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, localdir) else: workdir = os.path.dirname(os.path.realpath(moddir)) cmd = "tar %s -czf %s %s" % (tar_flags, ud.localpath, os.path.basename(moddir)) runfetchcmd(cmd, d, cleanup=[ud.localpath], workdir=workdir)