def test_get_installed(isolated_pio_core, tmpdir_factory): storage_dir = tmpdir_factory.mktemp("storage") pm = ToolPackageManager(str(storage_dir)) # VCS package (storage_dir.join("pkg-vcs").mkdir().join(".git").mkdir().join( ".piopm").write(""" { "name": "pkg-via-vcs", "spec": { "id": null, "name": "pkg-via-vcs", "owner": null, "requirements": null, "url": "git+https://github.com/username/repo.git" }, "type": "tool", "version": "0.0.0+sha.1ea4d5e" } """)) # package without metadata file (storage_dir.join("[email protected]").mkdir().join("package.json").write( '{"name": "foo", "version": "3.4.5"}')) # package with metadata file foo_dir = storage_dir.join("foo").mkdir() foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}') foo_dir.join(".piopm").write(""" { "name": "foo", "spec": { "name": "foo", "owner": null, "requirements": "^3" }, "type": "tool", "version": "3.6.0" } """) # test "system" storage_dir.join("pkg-incompatible-system").mkdir( ).join("package.json").write( '{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}') storage_dir.join("pkg-compatible-system").mkdir().join( "package.json").write( '{"name": "check-system", "version": "3.0.0", "system": "%s"}' % util.get_systype()) # invalid package storage_dir.join("invalid-package").mkdir().join("library.json").write( '{"name": "SomeLib", "version": "4.0.0"}') installed = pm.get_installed() assert len(installed) == 4 assert set(["pkg-via-vcs", "foo", "check-system"]) == set(p.metadata.name for p in installed) assert str(pm.get_package("foo").metadata.version) == "3.6.0" assert str(pm.get_package("check-system").metadata.version) == "3.0.0"
def remove_unnecessary_platform_packages(dry_run=False): candidates = [] required = set() core_packages = get_installed_core_packages() for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for pkg in p.get_installed_packages(with_optional=True): required.add(pkg) pm = ToolPackageManager() for pkg in pm.get_installed(): skip_conds = [ pkg.metadata.spec.url, os.path.isfile(os.path.join(pkg.path, ".piokeep")), pkg in required, pkg in core_packages, ] if not any(skip_conds): candidates.append(pkg) if dry_run: return candidates for pkg in candidates: pm.uninstall(pkg) return candidates
def cleanup_packages(self, names): self.memcache_reset() deppkgs = {} for platform in PlatformPackageManager().get_installed(): p = PlatformFactory.new(platform) for pkg in p.get_installed_packages(): if pkg.metadata.name not in deppkgs: deppkgs[pkg.metadata.name] = set() deppkgs[pkg.metadata.name].add(pkg.metadata.version) pm = ToolPackageManager() for pkg in pm.get_installed(): if pkg.metadata.name not in names: continue if ( pkg.metadata.name not in deppkgs or pkg.metadata.version not in deppkgs[pkg.metadata.name] ): try: pm.uninstall(pkg.metadata.spec) except UnknownPackageError: pass self.memcache_reset() return True
def get_installed_core_packages(): result = [] pm = ToolPackageManager() for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if pkg: result.append(pkg) return result
def test_install_from_registry(isolated_pio_core, tmpdir_factory): # Libraries lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage"))) # library with dependencies lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True) assert len(lm.get_installed()) == 3 pkg = lm.get_package("AsyncTCP-esphome") assert pkg.metadata.spec.owner == "ottowinter" assert not lm.get_package("non-existing-package") # mbed library assert lm.install("wolfSSL", silent=True) assert len(lm.get_installed()) == 4 # case sensitive author name assert lm.install("DallasTemperature", silent=True) assert lm.get_package("OneWire").metadata.version.major >= 2 assert len(lm.get_installed()) == 6 # test conflicted names lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage"))) lm.install("[email protected]", silent=True) lm.install("[email protected]", silent=True) assert len(lm.get_installed()) == 2 # Tools tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage"))) pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True) manifest = tm.load_manifest(pkg) assert tm.is_system_compatible(manifest.get("system")) assert util.get_systype() in manifest.get("system", []) # Test unknown with pytest.raises(UnknownPackageError): tm.install("unknown-package-tool @ 9.1.1", silent=True) with pytest.raises(UnknownPackageError): tm.install("owner/unknown-package-tool", silent=True)
def __init__(self, manifest_path): self.manifest_path = manifest_path self.silent = False self.verbose = False self._manifest = fs.load_json(manifest_path) self._BOARDS_CACHE = {} self._custom_packages = None self.config = ProjectConfig.get_instance() self.pm = ToolPackageManager(self.config.get_optional_dir("packages"))
def get_core_package_dir(name): if name not in __core_packages__: raise exception.PlatformioException("Please upgrade PlatformIO Core") pm = ToolPackageManager() spec = PackageSpec(owner="platformio", name=name, requirements=__core_packages__[name]) pkg = pm.get_package(spec) if pkg: return pkg.path assert pm.install(spec) _remove_unnecessary_packages() return pm.get_package(spec).path
def _update_pkg_metadata(_): pm = ToolPackageManager() for pkg in pm.get_installed(): if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id: continue result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name)) if len(result) != 1: continue result = result[0] pkg.metadata.spec = PackageSpec( id=result["id"], owner=result["owner"]["username"], name=result["name"], ) pkg.dump_meta() return True
def inject_contrib_pysite(verify_openssl=False): # pylint: disable=import-outside-toplevel from site import addsitedir try: contrib_pysite_dir = get_core_package_dir("contrib-pysite") except UnknownPackageError: pm = ToolPackageManager() contrib_pysite_dir = build_contrib_pysite_package( os.path.join(pm.package_dir, "contrib-pysite") ) if contrib_pysite_dir in sys.path: return True addsitedir(contrib_pysite_dir) sys.path.insert(0, contrib_pysite_dir) if not verify_openssl: return True try: # pylint: disable=import-error,unused-import,unused-variable from OpenSSL import SSL except: # pylint: disable=bare-except build_contrib_pysite_package(contrib_pysite_dir) return True
def system_info(json_output): project_config = ProjectConfig() data = {} data["core_version"] = {"title": "PlatformIO Core", "value": __version__} data["python_version"] = { "title": "Python", "value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)), } data["system"] = {"title": "System Type", "value": util.get_systype()} data["platform"] = { "title": "Platform", "value": platform.platform(terse=True) } data["filesystem_encoding"] = { "title": "File System Encoding", "value": compat.get_filesystem_encoding(), } data["locale_encoding"] = { "title": "Locale Encoding", "value": compat.get_locale_encoding(), } data["core_dir"] = { "title": "PlatformIO Core Directory", "value": project_config.get_optional_dir("core"), } data["platformio_exe"] = { "title": "PlatformIO Core Executable", "value": proc.where_is_program( "platformio.exe" if proc.WINDOWS else "platformio"), } data["python_exe"] = { "title": "Python Executable", "value": proc.get_pythonexe_path(), } data["global_lib_nums"] = { "title": "Global Libraries", "value": len(LibraryPackageManager().get_installed()), } data["dev_platform_nums"] = { "title": "Development Platforms", "value": len(PlatformPackageManager().get_installed()), } data["package_tool_nums"] = { "title": "Tools & Toolchains", "value": len( ToolPackageManager( project_config.get_optional_dir("packages")).get_installed()), } click.echo( json.dumps(data) if json_output else tabulate([( item["title"], item["value"]) for item in data.values()]))
def build_contrib_pysite_deps(target_dir): if os.path.isdir(target_dir): fs.rmtree(target_dir) os.makedirs(target_dir) # build dependencies pythonexe = get_pythonexe_path() for dep in get_contrib_pysite_deps(): subprocess.check_call([ pythonexe, "-m", "pip", "install", # "--no-cache-dir", "--no-compile", "--no-binary", ":all:", "-t", target_dir, dep, ]) # build manifests with open(os.path.join(target_dir, "package.json"), "w") as fp: json.dump( dict( name="contrib-pysite", version="2.%d%d.%s" % ( sys.version_info.major, sys.version_info.minor, date.today().strftime("%y%m%d"), ), system=util.get_systype(), ), fp, ) pm = ToolPackageManager() pkg = PackageItem(target_dir) pkg.metadata = pm.build_metadata( target_dir, PackageSpec(owner="platformio", name="contrib-pysite")) pkg.dump_meta() return True
def remove_unnecessary_core_packages(dry_run=False): candidates = [] pm = ToolPackageManager() best_pkg_versions = {} for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue best_pkg_versions[pkg.metadata.name] = pkg.metadata.version for pkg in pm.get_installed(): skip_conds = [ os.path.isfile(os.path.join(pkg.path, ".piokeep")), pkg.metadata.spec.owner != "platformio", pkg.metadata.name not in best_pkg_versions, pkg.metadata.name in best_pkg_versions and pkg.metadata.version == best_pkg_versions[pkg.metadata.name], ] if not any(skip_conds): candidates.append(pkg) if dry_run: return candidates for pkg in candidates: pm.uninstall(pkg) return candidates
def update_core_packages(only_check=False, silent=False): pm = ToolPackageManager() for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue if not silent or pm.outdated(pkg, spec).is_outdated(): pm.update(pkg, spec, only_check=only_check) if not only_check: remove_unnecessary_core_packages() return True
def _remove_unnecessary_packages(): pm = ToolPackageManager() best_pkg_versions = {} for name, requirements in __core_packages__.items(): spec = PackageSpec(owner="platformio", name=name, requirements=requirements) pkg = pm.get_package(spec) if not pkg: continue best_pkg_versions[pkg.metadata.name] = pkg.metadata.version for pkg in pm.get_installed(): if pkg.metadata.name not in best_pkg_versions: continue if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]: pm.uninstall(pkg)
class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods PlatformPackagesMixin, PlatformRunMixin): CORE_SEMVER = pepver_to_semver(__version__) _BOARDS_CACHE = {} def __init__(self, manifest_path): self.manifest_path = manifest_path self.silent = False self.verbose = False self._manifest = fs.load_json(manifest_path) self._BOARDS_CACHE = {} self._custom_packages = None self.config = ProjectConfig.get_instance() self.pm = ToolPackageManager(self.config.get_optional_dir("packages")) @property def name(self): return self._manifest["name"] @property def title(self): return self._manifest["title"] @property def description(self): return self._manifest["description"] @property def version(self): return self._manifest["version"] @property def homepage(self): return self._manifest.get("homepage") @property def repository_url(self): return self._manifest.get("repository", {}).get("url") @property def license(self): return self._manifest.get("license") @property def frameworks(self): return self._manifest.get("frameworks") @property def engines(self): return self._manifest.get("engines") @property def manifest(self): return self._manifest @property def packages(self): packages = self._manifest.get("packages", {}) for item in self._custom_packages or []: name = item version = "*" if "@" in item: name, version = item.split("@", 1) spec = self.pm.ensure_spec(name) options = {"version": version.strip(), "optional": False} if spec.owner: options["owner"] = spec.owner if spec.name not in packages: packages[spec.name] = {} packages[spec.name].update(**options) return packages @property def python_packages(self): return self._manifest.get("pythonPackages") def ensure_engine_compatible(self): if not self.engines or "platformio" not in self.engines: return True core_spec = semantic_version.SimpleSpec(self.engines["platformio"]) if self.CORE_SEMVER in core_spec: return True # PIO Core 5 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0 if any( semantic_version.Version.coerce(str(v)) in core_spec for v in (2, 3, 4)): return True raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER), str(core_spec)) def get_dir(self): return os.path.dirname(self.manifest_path) def get_build_script(self): main_script = os.path.join(self.get_dir(), "builder", "main.py") if os.path.isfile(main_script): return main_script raise NotImplementedError() def is_embedded(self): for opts in self.packages.values(): if opts.get("type") == "uploader": return True return False def get_boards(self, id_=None): def _append_board(board_id, manifest_path): config = PlatformBoardConfig(manifest_path) if "platform" in config and config.get("platform") != self.name: return if "platforms" in config and self.name not in config.get( "platforms"): return config.manifest["platform"] = self.name self._BOARDS_CACHE[board_id] = config bdirs = [ self.config.get_optional_dir("boards"), os.path.join(self.config.get_optional_dir("core"), "boards"), os.path.join(self.get_dir(), "boards"), ] if id_ is None: for boards_dir in bdirs: if not os.path.isdir(boards_dir): continue for item in sorted(os.listdir(boards_dir)): _id = item[:-5] if not item.endswith(".json") or _id in self._BOARDS_CACHE: continue _append_board(_id, os.path.join(boards_dir, item)) else: if id_ not in self._BOARDS_CACHE: for boards_dir in bdirs: if not os.path.isdir(boards_dir): continue manifest_path = os.path.join(boards_dir, "%s.json" % id_) if os.path.isfile(manifest_path): _append_board(id_, manifest_path) break if id_ not in self._BOARDS_CACHE: raise UnknownBoard(id_) return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE def board_config(self, id_): return self.get_boards(id_) def get_package_type(self, name): return self.packages[name].get("type") def configure_default_packages(self, options, targets): # override user custom packages self._custom_packages = options.get("platform_packages") # enable used frameworks for framework in options.get("framework", []): if not self.frameworks: continue framework = framework.lower().strip() if not framework or framework not in self.frameworks: continue _pkg_name = self.frameworks[framework].get("package") if _pkg_name: self.packages[_pkg_name]["optional"] = False # enable upload tools for upload targets if any(["upload" in t for t in targets] + ["program" in targets]): for name, opts in self.packages.items(): if opts.get("type") == "uploader": self.packages[name]["optional"] = False # skip all packages in "nobuild" mode # allow only upload tools and frameworks elif "nobuild" in targets and opts.get("type") != "framework": self.packages[name]["optional"] = True def configure_debug_options(self, initial_debug_options, ide_data): raise NotImplementedError def get_lib_storages(self): storages = {} for opts in (self.frameworks or {}).values(): if "package" not in opts: continue pkg = self.get_package(opts["package"]) if not pkg or not os.path.isdir(os.path.join( pkg.path, "libraries")): continue libs_dir = os.path.join(pkg.path, "libraries") storages[libs_dir] = opts["package"] libcores_dir = os.path.join(libs_dir, "__cores__") if not os.path.isdir(libcores_dir): continue for item in os.listdir(libcores_dir): libcore_dir = os.path.join(libcores_dir, item) if not os.path.isdir(libcore_dir): continue storages[libcore_dir] = "%s-core-%s" % (opts["package"], item) return [dict(name=name, path=path) for path, name in storages.items()] def on_installed(self): pass def on_uninstalled(self): pass def install_python_packages(self): if not self.python_packages: return None click.echo( "Installing Python packages: %s" % ", ".join(list(self.python_packages.keys())), ) args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"] for name, requirements in self.python_packages.items(): if any(c in requirements for c in ("<", ">", "=")): args.append("%s%s" % (name, requirements)) else: args.append("%s==%s" % (name, requirements)) try: return subprocess.call(args) == 0 except Exception as e: # pylint: disable=broad-except click.secho("Could not install Python packages -> %s" % e, fg="red", err=True) return None def uninstall_python_packages(self): if not self.python_packages: return click.echo("Uninstalling Python packages") args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"] args.extend(list(self.python_packages.keys())) try: subprocess.call(args) == 0 except Exception as e: # pylint: disable=broad-except click.secho("Could not install Python packages -> %s" % e, fg="red", err=True)
def build_contrib_pysite_package(target_dir, with_metadata=True): systype = util.get_systype() if os.path.isdir(target_dir): fs.rmtree(target_dir) os.makedirs(target_dir) # build dependencies args = [ get_pythonexe_path(), "-m", "pip", "install", "--no-compile", "-t", target_dir, ] if "linux" in systype: args.extend(["--no-binary", ":all:"]) for dep in get_contrib_pysite_deps(): subprocess.check_call(args + [dep]) # build manifests with open(os.path.join(target_dir, "package.json"), "w") as fp: json.dump( dict( name="contrib-pysite", version="2.%d%d.%s" % ( sys.version_info.major, sys.version_info.minor, date.today().strftime("%y%m%d"), ), system=list( set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"]) ) if systype.startswith("linux_arm") else systype, description="Extra Python package for PlatformIO Core", keywords=["platformio", "platformio-core"], homepage="https://docs.platformio.org/page/core/index.html", repository={ "type": "git", "url": "https://github.com/platformio/platformio-core", }, ), fp, ) # generate package metadata if with_metadata: pm = ToolPackageManager() pkg = PackageItem(target_dir) pkg.metadata = pm.build_metadata( target_dir, PackageSpec(owner="platformio", name="contrib-pysite") ) pkg.dump_meta() # remove unused files for root, dirs, files in os.walk(target_dir): for t in ("_test", "test", "tests"): if t in dirs: shutil.rmtree(os.path.join(root, t)) for name in files: if name.endswith((".chm", ".pyc")): os.remove(os.path.join(root, name)) return target_dir