示例#1
0
    def urldata_init(self, ud, d):
        """
        init NPM specific variable within url data
        """
        if 'downloadfilename' in ud.parm:
            ud.basename = ud.parm['downloadfilename']
        else:
            ud.basename = os.path.basename(ud.path)

        # can't call it ud.name otherwise fetcher base class will start doing sha1stuff
        # TODO: find a way to get an sha1/sha256 manifest of pkg & all deps
        ud.pkgname = ud.parm.get("name", None)
        if not ud.pkgname:
            raise ParameterError("NPM fetcher requires a name parameter", ud.url)
        ud.version = ud.parm.get("version", None)
        if not ud.version:
            raise ParameterError("NPM fetcher requires a version parameter", ud.url)
        ud.bbnpmmanifest = "%s-%s.deps.json" % (ud.pkgname, ud.version)
        ud.registry = "http://%s" % (ud.url.replace('npm://', '', 1).split(';'))[0]
        prefixdir = "npm/%s" % ud.pkgname
        ud.pkgdatadir = d.expand("${DL_DIR}/%s" % prefixdir)
        if not os.path.exists(ud.pkgdatadir):
            bb.utils.mkdirhier(ud.pkgdatadir)
        ud.localpath = d.expand("${DL_DIR}/npm/%s" % ud.bbnpmmanifest)

        self.basecmd = d.getVar("FETCHCMD_wget", True) or "/usr/bin/env wget -O -t 2 -T 30 -nv --passive-ftp --no-check-certificate "
        ud.prefixdir = prefixdir

        ud.write_tarballs = ((data.getVar("BB_GENERATE_MIRROR_TARBALLS", d, True) or "0") != "0")
        ud.mirrortarball = 'npm_%s-%s.tar.xz' % (ud.pkgname, ud.version)
        ud.fullmirror = os.path.join(d.getVar("DL_DIR", True), ud.mirrortarball)
示例#2
0
        def _npm_view():
            args = []
            args.append(("json", "true"))
            args.append(("registry", ud.registry))
            pkgver = shlex.quote(ud.package + "@" + ud.version)
            cmd = ud.basecmd + " view %s" % pkgver
            env = NpmEnvironment(d)
            check_network_access(d, cmd, ud.registry)
            view_string = env.run(cmd, args=args)

            if not view_string:
                raise FetchError("Unavailable package %s" % pkgver, ud.url)

            try:
                view = json.loads(view_string)

                error = view.get("error")
                if error is not None:
                    raise FetchError(error.get("summary"), ud.url)

                if ud.version == "latest":
                    bb.warn("The npm package %s is using the latest " \
                            "version available. This could lead to " \
                            "non-reproducible builds." % pkgver)
                elif ud.version != view.get("version"):
                    raise ParameterError("Invalid 'version' parameter", ud.url)

                return view

            except Exception as e:
                raise FetchError("Invalid view from npm: %s" % str(e), ud.url)
示例#3
0
    def urldata_init(self, ud, d):
        """
        init ClearCase specific variable within url data
        """
        ud.proto = "https"
        if 'protocol' in ud.parm:
            ud.proto = ud.parm['protocol']
        if not ud.proto in ('http', 'https'):
            raise ParameterError("Invalid protocol type", ud.url)

        ud.vob = ''
        if 'vob' in ud.parm:
            ud.vob = ud.parm['vob']
        else:
            msg = ud.url+": vob must be defined so the fetcher knows what to get."
            raise MissingParameterError('vob', msg)

        if 'module' in ud.parm:
            ud.module = ud.parm['module']
        else:
            ud.module = ""

        ud.basecmd = d.getVar("FETCHCMD_ccrc") or "/usr/bin/env cleartool || rcleartool"

        if d.getVar("SRCREV") == "INVALID":
          raise FetchError("Set a valid SRCREV for the clearcase fetcher in your recipe, e.g. SRCREV = \"/main/LATEST\" or any other label of your choice.")

        ud.label = d.getVar("SRCREV", False)
        ud.customspec = d.getVar("CCASE_CUSTOM_CONFIG_SPEC")

        ud.server     = "%s://%s%s" % (ud.proto, ud.host, ud.path)

        ud.identifier = "clearcase-%s%s-%s" % ( ud.vob.replace("/", ""),
                                                ud.module.replace("/", "."),
                                                ud.label.replace("/", "."))

        ud.viewname         = "%s-view%s" % (ud.identifier, d.getVar("DATETIME", d, True))
        ud.csname           = "%s-config-spec" % (ud.identifier)
        ud.ccasedir         = os.path.join(d.getVar("DL_DIR"), ud.type)
        ud.viewdir          = os.path.join(ud.ccasedir, ud.viewname)
        ud.configspecfile   = os.path.join(ud.ccasedir, ud.csname)
        ud.localfile        = "%s.tar.gz" % (ud.identifier)

        self.debug("host            = %s" % ud.host)
        self.debug("path            = %s" % ud.path)
        self.debug("server          = %s" % ud.server)
        self.debug("proto           = %s" % ud.proto)
        self.debug("type            = %s" % ud.type)
        self.debug("vob             = %s" % ud.vob)
        self.debug("module          = %s" % ud.module)
        self.debug("basecmd         = %s" % ud.basecmd)
        self.debug("label           = %s" % ud.label)
        self.debug("ccasedir        = %s" % ud.ccasedir)
        self.debug("viewdir         = %s" % ud.viewdir)
        self.debug("viewname        = %s" % ud.viewname)
        self.debug("configspecfile  = %s" % ud.configspecfile)
        self.debug("localfile       = %s" % ud.localfile)

        ud.localfile = os.path.join(d.getVar("DL_DIR"), ud.localfile)
示例#4
0
 def urldata_init(self, urldata, d):
     if 'protocol' in urldata.parm and urldata.parm['protocol'] == 'git':
         raise ParameterError(
             "Invalid protocol - if you wish to fetch from a git " +
             "repository using ssh, you need to use " +
             "git:// prefix with protocol=ssh", urldata.url)
     m = __pattern__.match(urldata.url)
     path = m.group('path')
     host = m.group('host')
     urldata.localpath = os.path.join(
         d.getVar('DL_DIR'), os.path.basename(os.path.normpath(path)))
示例#5
0
    def urldata_init(self, ud, d):
        """Init npm specific variables within url data"""
        ud.package = None
        ud.version = None
        ud.registry = None

        # Get the 'package' parameter
        if "package" in ud.parm:
            ud.package = ud.parm.get("package")

        if not ud.package:
            raise MissingParameterError("Parameter 'package' required", ud.url)

        # Get the 'version' parameter
        if "version" in ud.parm:
            ud.version = ud.parm.get("version")

        if not ud.version:
            raise MissingParameterError("Parameter 'version' required", ud.url)

        if not is_semver(ud.version) and not ud.version == "latest":
            raise ParameterError("Invalid 'version' parameter", ud.url)

        # Extract the 'registry' part of the url
        ud.registry = re.sub(r"^npm://", "http://", ud.url.split(";")[0])

        # Using the 'downloadfilename' parameter as local filename
        # or the npm package name.
        if "downloadfilename" in ud.parm:
            ud.localfile = npm_localfile(d.expand(ud.parm["downloadfilename"]))
        else:
            ud.localfile = npm_localfile(ud.package, ud.version)

        # Get the base 'npm' command
        ud.basecmd = d.getVar("FETCHCMD_npm") or "npm"

        # This fetcher resolves a URI from a npm package name and version and
        # then forwards it to a proxy fetcher. A resolve file containing the
        # resolved URI is created to avoid unwanted network access (if the file
        # already exists). The management of the donestamp file, the lockfile
        # and the checksums are forwarded to the proxy fetcher.
        ud.proxy = None
        ud.needdonestamp = False
        ud.resolvefile = self.localpath(ud, d) + ".resolved"
示例#6
0
        def _resolve_dependency(name, params, deptree):
            url = None
            localpath = None
            extrapaths = []
            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
            destsuffix = os.path.join(*destsubdirs)
            unpack = True

            integrity = params.get("integrity", None)
            resolved = params.get("resolved", None)
            version = params.get("version", None)

            # Handle registry sources
            if is_semver(version) and integrity:
                # Handle duplicate dependencies without url
                if not resolved:
                    return

                localfile = npm_localfile(name, version)

                uri = URI(resolved)
                uri.params["downloadfilename"] = localfile

                checksum_name, checksum_expected = npm_integrity(integrity)
                uri.params[checksum_name] = checksum_expected

                url = str(uri)

                localpath = os.path.join(d.getVar("DL_DIR"), localfile)

                # Create a resolve file to mimic the npm fetcher and allow
                # re-usability of the downloaded file.
                resolvefile = localpath + ".resolved"

                bb.utils.mkdirhier(os.path.dirname(resolvefile))
                with open(resolvefile, "w") as f:
                    f.write(url)

                extrapaths.append(resolvefile)

            # Handle http tarball sources
            elif version.startswith("http") and integrity:
                localfile = npm_localfile(os.path.basename(version))

                uri = URI(version)
                uri.params["downloadfilename"] = localfile

                checksum_name, checksum_expected = npm_integrity(integrity)
                uri.params[checksum_name] = checksum_expected

                url = str(uri)

                localpath = os.path.join(d.getVar("DL_DIR"), localfile)

            # Handle git sources
            elif version.startswith("git"):
                if version.startswith("github:"):
                    version = "git+https://github.com/" + version[len("github:"):]
                regex = re.compile(r"""
                    ^
                    git\+
                    (?P<protocol>[a-z]+)
                    ://
                    (?P<url>[^#]+)
                    \#
                    (?P<rev>[0-9a-f]+)
                    $
                    """, re.VERBOSE)

                match = regex.match(version)

                if not match:
                    raise ParameterError("Invalid git url: %s" % version, ud.url)

                groups = match.groupdict()

                uri = URI("git://" + str(groups["url"]))
                uri.params["protocol"] = str(groups["protocol"])
                uri.params["rev"] = str(groups["rev"])
                uri.params["destsuffix"] = destsuffix

                url = str(uri)

            # Handle local tarball and link sources
            elif version.startswith("file"):
                localpath = version[5:]
                if not version.endswith(".tgz"):
                    unpack = False

            else:
                raise ParameterError("Unsupported dependency: %s" % name, ud.url)

            ud.deps.append({
                "url": url,
                "localpath": localpath,
                "extrapaths": extrapaths,
                "destsuffix": destsuffix,
                "unpack": unpack,
            })
示例#7
0
    def urldata_init(self, ud, d):
        """Init npmsw specific variables within url data"""

        # Get the 'shrinkwrap' parameter
        ud.shrinkwrap_file = re.sub(r"^npmsw://", "", ud.url.split(";")[0])

        # Get the 'dev' parameter
        ud.dev = bb.utils.to_boolean(ud.parm.get("dev"), False)

        # Resolve the dependencies
        ud.deps = []

        def _resolve_dependency(name, params, deptree):
            url = None
            localpath = None
            extrapaths = []
            destsubdirs = [os.path.join("node_modules", dep) for dep in deptree]
            destsuffix = os.path.join(*destsubdirs)
            unpack = True

            integrity = params.get("integrity", None)
            resolved = params.get("resolved", None)
            version = params.get("version", None)

            # Handle registry sources
            if is_semver(version) and integrity:
                # Handle duplicate dependencies without url
                if not resolved:
                    return

                localfile = npm_localfile(name, version)

                uri = URI(resolved)
                uri.params["downloadfilename"] = localfile

                checksum_name, checksum_expected = npm_integrity(integrity)
                uri.params[checksum_name] = checksum_expected

                url = str(uri)

                localpath = os.path.join(d.getVar("DL_DIR"), localfile)

                # Create a resolve file to mimic the npm fetcher and allow
                # re-usability of the downloaded file.
                resolvefile = localpath + ".resolved"

                bb.utils.mkdirhier(os.path.dirname(resolvefile))
                with open(resolvefile, "w") as f:
                    f.write(url)

                extrapaths.append(resolvefile)

            # Handle http tarball sources
            elif version.startswith("http") and integrity:
                localfile = npm_localfile(os.path.basename(version))

                uri = URI(version)
                uri.params["downloadfilename"] = localfile

                checksum_name, checksum_expected = npm_integrity(integrity)
                uri.params[checksum_name] = checksum_expected

                url = str(uri)

                localpath = os.path.join(d.getVar("DL_DIR"), localfile)

            # Handle git sources
            elif version.startswith("git"):
                if version.startswith("github:"):
                    version = "git+https://github.com/" + version[len("github:"):]
                regex = re.compile(r"""
                    ^
                    git\+
                    (?P<protocol>[a-z]+)
                    ://
                    (?P<url>[^#]+)
                    \#
                    (?P<rev>[0-9a-f]+)
                    $
                    """, re.VERBOSE)

                match = regex.match(version)

                if not match:
                    raise ParameterError("Invalid git url: %s" % version, ud.url)

                groups = match.groupdict()

                uri = URI("git://" + str(groups["url"]))
                uri.params["protocol"] = str(groups["protocol"])
                uri.params["rev"] = str(groups["rev"])
                uri.params["destsuffix"] = destsuffix

                url = str(uri)

            # Handle local tarball and link sources
            elif version.startswith("file"):
                localpath = version[5:]
                if not version.endswith(".tgz"):
                    unpack = False

            else:
                raise ParameterError("Unsupported dependency: %s" % name, ud.url)

            ud.deps.append({
                "url": url,
                "localpath": localpath,
                "extrapaths": extrapaths,
                "destsuffix": destsuffix,
                "unpack": unpack,
            })

        try:
            with open(ud.shrinkwrap_file, "r") as f:
                shrinkwrap = json.load(f)
        except Exception as e:
            raise ParameterError("Invalid shrinkwrap file: %s" % str(e), ud.url)

        foreach_dependencies(shrinkwrap, _resolve_dependency, ud.dev)

        # Avoid conflicts between the environment data and:
        # - the proxy url revision
        # - the proxy url checksum
        data = bb.data.createCopy(d)
        data.delVar("SRCREV")
        data.delVarFlags("SRC_URI")

        # This fetcher resolves multiple URIs from a shrinkwrap file and then
        # forwards it to a proxy fetcher. The management of the donestamp file,
        # the lockfile and the checksums are forwarded to the proxy fetcher.
        ud.proxy = Fetch([dep["url"] for dep in ud.deps if dep["url"]], data)
        ud.needdonestamp = False