def test_get_installed(isolated_pio_core, tmpdir_factory):
storage_dir = tmpdir_factory.mktemp("storage")
pm = ToolPackageManager(str(storage_dir))
# VCS package
(storage_dir.join("pkg-vcs").mkdir().join(".git").mkdir().join(
".piopm").write("""
{
"name": "pkg-via-vcs",
"spec": {
"id": null,
"name": "pkg-via-vcs",
"owner": null,
"requirements": null,
"url": "git+https://github.com/username/repo.git"
},
"type": "tool",
"version": "0.0.0+sha.1ea4d5e"
}
"""))
# package without metadata file
(storage_dir.join("[email protected]").mkdir().join("package.json").write(
'{"name": "foo", "version": "3.4.5"}'))
# package with metadata file
foo_dir = storage_dir.join("foo").mkdir()
foo_dir.join("package.json").write('{"name": "foo", "version": "3.6.0"}')
foo_dir.join(".piopm").write("""
{
"name": "foo",
"spec": {
"name": "foo",
"owner": null,
"requirements": "^3"
},
"type": "tool",
"version": "3.6.0"
}
""")
# test "system"
storage_dir.join("pkg-incompatible-system").mkdir(
).join("package.json").write(
'{"name": "check-system", "version": "4.0.0", "system": ["unknown"]}')
storage_dir.join("pkg-compatible-system").mkdir().join(
"package.json").write(
'{"name": "check-system", "version": "3.0.0", "system": "%s"}' %
util.get_systype())
# invalid package
storage_dir.join("invalid-package").mkdir().join("library.json").write(
'{"name": "SomeLib", "version": "4.0.0"}')
installed = pm.get_installed()
assert len(installed) == 4
assert set(["pkg-via-vcs", "foo",
"check-system"]) == set(p.metadata.name for p in installed)
assert str(pm.get_package("foo").metadata.version) == "3.6.0"
assert str(pm.get_package("check-system").metadata.version) == "3.0.0"
def remove_unnecessary_platform_packages(dry_run=False):
candidates = []
required = set()
core_packages = get_installed_core_packages()
for platform in PlatformPackageManager().get_installed():
p = PlatformFactory.new(platform)
for pkg in p.get_installed_packages(with_optional=True):
required.add(pkg)
pm = ToolPackageManager()
for pkg in pm.get_installed():
skip_conds = [
pkg.metadata.spec.url,
os.path.isfile(os.path.join(pkg.path, ".piokeep")),
pkg in required,
pkg in core_packages,
]
if not any(skip_conds):
candidates.append(pkg)
if dry_run:
return candidates
for pkg in candidates:
pm.uninstall(pkg)
return candidates
def cleanup_packages(self, names):
self.memcache_reset()
deppkgs = {}
for platform in PlatformPackageManager().get_installed():
p = PlatformFactory.new(platform)
for pkg in p.get_installed_packages():
if pkg.metadata.name not in deppkgs:
deppkgs[pkg.metadata.name] = set()
deppkgs[pkg.metadata.name].add(pkg.metadata.version)
pm = ToolPackageManager()
for pkg in pm.get_installed():
if pkg.metadata.name not in names:
continue
if (
pkg.metadata.name not in deppkgs
or pkg.metadata.version not in deppkgs[pkg.metadata.name]
):
try:
pm.uninstall(pkg.metadata.spec)
except UnknownPackageError:
pass
self.memcache_reset()
return True
def get_installed_core_packages():
result = []
pm = ToolPackageManager()
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if pkg:
result.append(pkg)
return result
def test_install_from_registry(isolated_pio_core, tmpdir_factory):
# Libraries
lm = LibraryPackageManager(str(tmpdir_factory.mktemp("lib-storage")))
# library with dependencies
lm.install("AsyncMqttClient-esphome @ 0.8.4", silent=True)
assert len(lm.get_installed()) == 3
pkg = lm.get_package("AsyncTCP-esphome")
assert pkg.metadata.spec.owner == "ottowinter"
assert not lm.get_package("non-existing-package")
# mbed library
assert lm.install("wolfSSL", silent=True)
assert len(lm.get_installed()) == 4
# case sensitive author name
assert lm.install("DallasTemperature", silent=True)
assert lm.get_package("OneWire").metadata.version.major >= 2
assert len(lm.get_installed()) == 6
# test conflicted names
lm = LibraryPackageManager(str(tmpdir_factory.mktemp("conflicted-storage")))
lm.install("[email protected]", silent=True)
lm.install("[email protected]", silent=True)
assert len(lm.get_installed()) == 2
# Tools
tm = ToolPackageManager(str(tmpdir_factory.mktemp("tool-storage")))
pkg = tm.install("platformio/tool-stlink @ ~1.10400.0", silent=True)
manifest = tm.load_manifest(pkg)
assert tm.is_system_compatible(manifest.get("system"))
assert util.get_systype() in manifest.get("system", [])
# Test unknown
with pytest.raises(UnknownPackageError):
tm.install("unknown-package-tool @ 9.1.1", silent=True)
with pytest.raises(UnknownPackageError):
tm.install("owner/unknown-package-tool", silent=True)
def __init__(self, manifest_path):
self.manifest_path = manifest_path
self.silent = False
self.verbose = False
self._manifest = fs.load_json(manifest_path)
self._BOARDS_CACHE = {}
self._custom_packages = None
self.config = ProjectConfig.get_instance()
self.pm = ToolPackageManager(self.config.get_optional_dir("packages"))
def get_core_package_dir(name):
if name not in __core_packages__:
raise exception.PlatformioException("Please upgrade PlatformIO Core")
pm = ToolPackageManager()
spec = PackageSpec(owner="platformio",
name=name,
requirements=__core_packages__[name])
pkg = pm.get_package(spec)
if pkg:
return pkg.path
assert pm.install(spec)
_remove_unnecessary_packages()
return pm.get_package(spec).path
def _update_pkg_metadata(_):
pm = ToolPackageManager()
for pkg in pm.get_installed():
if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id:
continue
result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name))
if len(result) != 1:
continue
result = result[0]
pkg.metadata.spec = PackageSpec(
id=result["id"],
owner=result["owner"]["username"],
name=result["name"],
)
pkg.dump_meta()
return True
def inject_contrib_pysite(verify_openssl=False):
# pylint: disable=import-outside-toplevel
from site import addsitedir
try:
contrib_pysite_dir = get_core_package_dir("contrib-pysite")
except UnknownPackageError:
pm = ToolPackageManager()
contrib_pysite_dir = build_contrib_pysite_package(
os.path.join(pm.package_dir, "contrib-pysite")
)
if contrib_pysite_dir in sys.path:
return True
addsitedir(contrib_pysite_dir)
sys.path.insert(0, contrib_pysite_dir)
if not verify_openssl:
return True
try:
# pylint: disable=import-error,unused-import,unused-variable
from OpenSSL import SSL
except: # pylint: disable=bare-except
build_contrib_pysite_package(contrib_pysite_dir)
return True
def system_info(json_output):
project_config = ProjectConfig()
data = {}
data["core_version"] = {"title": "PlatformIO Core", "value": __version__}
data["python_version"] = {
"title": "Python",
"value": "{0}.{1}.{2}-{3}.{4}".format(*list(sys.version_info)),
}
data["system"] = {"title": "System Type", "value": util.get_systype()}
data["platform"] = {
"title": "Platform",
"value": platform.platform(terse=True)
}
data["filesystem_encoding"] = {
"title": "File System Encoding",
"value": compat.get_filesystem_encoding(),
}
data["locale_encoding"] = {
"title": "Locale Encoding",
"value": compat.get_locale_encoding(),
}
data["core_dir"] = {
"title": "PlatformIO Core Directory",
"value": project_config.get_optional_dir("core"),
}
data["platformio_exe"] = {
"title":
"PlatformIO Core Executable",
"value":
proc.where_is_program(
"platformio.exe" if proc.WINDOWS else "platformio"),
}
data["python_exe"] = {
"title": "Python Executable",
"value": proc.get_pythonexe_path(),
}
data["global_lib_nums"] = {
"title": "Global Libraries",
"value": len(LibraryPackageManager().get_installed()),
}
data["dev_platform_nums"] = {
"title": "Development Platforms",
"value": len(PlatformPackageManager().get_installed()),
}
data["package_tool_nums"] = {
"title":
"Tools & Toolchains",
"value":
len(
ToolPackageManager(
project_config.get_optional_dir("packages")).get_installed()),
}
click.echo(
json.dumps(data) if json_output else tabulate([(
item["title"], item["value"]) for item in data.values()]))
def build_contrib_pysite_deps(target_dir):
if os.path.isdir(target_dir):
fs.rmtree(target_dir)
os.makedirs(target_dir)
# build dependencies
pythonexe = get_pythonexe_path()
for dep in get_contrib_pysite_deps():
subprocess.check_call([
pythonexe,
"-m",
"pip",
"install",
# "--no-cache-dir",
"--no-compile",
"--no-binary",
":all:",
"-t",
target_dir,
dep,
])
# build manifests
with open(os.path.join(target_dir, "package.json"), "w") as fp:
json.dump(
dict(
name="contrib-pysite",
version="2.%d%d.%s" % (
sys.version_info.major,
sys.version_info.minor,
date.today().strftime("%y%m%d"),
),
system=util.get_systype(),
),
fp,
)
pm = ToolPackageManager()
pkg = PackageItem(target_dir)
pkg.metadata = pm.build_metadata(
target_dir, PackageSpec(owner="platformio", name="contrib-pysite"))
pkg.dump_meta()
return True
def remove_unnecessary_core_packages(dry_run=False):
candidates = []
pm = ToolPackageManager()
best_pkg_versions = {}
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
best_pkg_versions[pkg.metadata.name] = pkg.metadata.version
for pkg in pm.get_installed():
skip_conds = [
os.path.isfile(os.path.join(pkg.path, ".piokeep")),
pkg.metadata.spec.owner != "platformio",
pkg.metadata.name not in best_pkg_versions,
pkg.metadata.name in best_pkg_versions
and pkg.metadata.version == best_pkg_versions[pkg.metadata.name],
]
if not any(skip_conds):
candidates.append(pkg)
if dry_run:
return candidates
for pkg in candidates:
pm.uninstall(pkg)
return candidates
def update_core_packages(only_check=False, silent=False):
pm = ToolPackageManager()
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
if not silent or pm.outdated(pkg, spec).is_outdated():
pm.update(pkg, spec, only_check=only_check)
if not only_check:
remove_unnecessary_core_packages()
return True
def _remove_unnecessary_packages():
pm = ToolPackageManager()
best_pkg_versions = {}
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio",
name=name,
requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
best_pkg_versions[pkg.metadata.name] = pkg.metadata.version
for pkg in pm.get_installed():
if pkg.metadata.name not in best_pkg_versions:
continue
if pkg.metadata.version != best_pkg_versions[pkg.metadata.name]:
pm.uninstall(pkg)
class PlatformBase( # pylint: disable=too-many-instance-attributes,too-many-public-methods
PlatformPackagesMixin, PlatformRunMixin):
CORE_SEMVER = pepver_to_semver(__version__)
_BOARDS_CACHE = {}
def __init__(self, manifest_path):
self.manifest_path = manifest_path
self.silent = False
self.verbose = False
self._manifest = fs.load_json(manifest_path)
self._BOARDS_CACHE = {}
self._custom_packages = None
self.config = ProjectConfig.get_instance()
self.pm = ToolPackageManager(self.config.get_optional_dir("packages"))
@property
def name(self):
return self._manifest["name"]
@property
def title(self):
return self._manifest["title"]
@property
def description(self):
return self._manifest["description"]
@property
def version(self):
return self._manifest["version"]
@property
def homepage(self):
return self._manifest.get("homepage")
@property
def repository_url(self):
return self._manifest.get("repository", {}).get("url")
@property
def license(self):
return self._manifest.get("license")
@property
def frameworks(self):
return self._manifest.get("frameworks")
@property
def engines(self):
return self._manifest.get("engines")
@property
def manifest(self):
return self._manifest
@property
def packages(self):
packages = self._manifest.get("packages", {})
for item in self._custom_packages or []:
name = item
version = "*"
if "@" in item:
name, version = item.split("@", 1)
spec = self.pm.ensure_spec(name)
options = {"version": version.strip(), "optional": False}
if spec.owner:
options["owner"] = spec.owner
if spec.name not in packages:
packages[spec.name] = {}
packages[spec.name].update(**options)
return packages
@property
def python_packages(self):
return self._manifest.get("pythonPackages")
def ensure_engine_compatible(self):
if not self.engines or "platformio" not in self.engines:
return True
core_spec = semantic_version.SimpleSpec(self.engines["platformio"])
if self.CORE_SEMVER in core_spec:
return True
# PIO Core 5 is compatible with dev-platforms for PIO Core 2.0, 3.0, 4.0
if any(
semantic_version.Version.coerce(str(v)) in core_spec
for v in (2, 3, 4)):
return True
raise IncompatiblePlatform(self.name, str(self.CORE_SEMVER),
str(core_spec))
def get_dir(self):
return os.path.dirname(self.manifest_path)
def get_build_script(self):
main_script = os.path.join(self.get_dir(), "builder", "main.py")
if os.path.isfile(main_script):
return main_script
raise NotImplementedError()
def is_embedded(self):
for opts in self.packages.values():
if opts.get("type") == "uploader":
return True
return False
def get_boards(self, id_=None):
def _append_board(board_id, manifest_path):
config = PlatformBoardConfig(manifest_path)
if "platform" in config and config.get("platform") != self.name:
return
if "platforms" in config and self.name not in config.get(
"platforms"):
return
config.manifest["platform"] = self.name
self._BOARDS_CACHE[board_id] = config
bdirs = [
self.config.get_optional_dir("boards"),
os.path.join(self.config.get_optional_dir("core"), "boards"),
os.path.join(self.get_dir(), "boards"),
]
if id_ is None:
for boards_dir in bdirs:
if not os.path.isdir(boards_dir):
continue
for item in sorted(os.listdir(boards_dir)):
_id = item[:-5]
if not item.endswith(".json") or _id in self._BOARDS_CACHE:
continue
_append_board(_id, os.path.join(boards_dir, item))
else:
if id_ not in self._BOARDS_CACHE:
for boards_dir in bdirs:
if not os.path.isdir(boards_dir):
continue
manifest_path = os.path.join(boards_dir, "%s.json" % id_)
if os.path.isfile(manifest_path):
_append_board(id_, manifest_path)
break
if id_ not in self._BOARDS_CACHE:
raise UnknownBoard(id_)
return self._BOARDS_CACHE[id_] if id_ else self._BOARDS_CACHE
def board_config(self, id_):
return self.get_boards(id_)
def get_package_type(self, name):
return self.packages[name].get("type")
def configure_default_packages(self, options, targets):
# override user custom packages
self._custom_packages = options.get("platform_packages")
# enable used frameworks
for framework in options.get("framework", []):
if not self.frameworks:
continue
framework = framework.lower().strip()
if not framework or framework not in self.frameworks:
continue
_pkg_name = self.frameworks[framework].get("package")
if _pkg_name:
self.packages[_pkg_name]["optional"] = False
# enable upload tools for upload targets
if any(["upload" in t for t in targets] + ["program" in targets]):
for name, opts in self.packages.items():
if opts.get("type") == "uploader":
self.packages[name]["optional"] = False
# skip all packages in "nobuild" mode
# allow only upload tools and frameworks
elif "nobuild" in targets and opts.get("type") != "framework":
self.packages[name]["optional"] = True
def configure_debug_options(self, initial_debug_options, ide_data):
raise NotImplementedError
def get_lib_storages(self):
storages = {}
for opts in (self.frameworks or {}).values():
if "package" not in opts:
continue
pkg = self.get_package(opts["package"])
if not pkg or not os.path.isdir(os.path.join(
pkg.path, "libraries")):
continue
libs_dir = os.path.join(pkg.path, "libraries")
storages[libs_dir] = opts["package"]
libcores_dir = os.path.join(libs_dir, "__cores__")
if not os.path.isdir(libcores_dir):
continue
for item in os.listdir(libcores_dir):
libcore_dir = os.path.join(libcores_dir, item)
if not os.path.isdir(libcore_dir):
continue
storages[libcore_dir] = "%s-core-%s" % (opts["package"], item)
return [dict(name=name, path=path) for path, name in storages.items()]
def on_installed(self):
pass
def on_uninstalled(self):
pass
def install_python_packages(self):
if not self.python_packages:
return None
click.echo(
"Installing Python packages: %s" %
", ".join(list(self.python_packages.keys())), )
args = [proc.get_pythonexe_path(), "-m", "pip", "install", "--upgrade"]
for name, requirements in self.python_packages.items():
if any(c in requirements for c in ("<", ">", "=")):
args.append("%s%s" % (name, requirements))
else:
args.append("%s==%s" % (name, requirements))
try:
return subprocess.call(args) == 0
except Exception as e: # pylint: disable=broad-except
click.secho("Could not install Python packages -> %s" % e,
fg="red",
err=True)
return None
def uninstall_python_packages(self):
if not self.python_packages:
return
click.echo("Uninstalling Python packages")
args = [proc.get_pythonexe_path(), "-m", "pip", "uninstall", "--yes"]
args.extend(list(self.python_packages.keys()))
try:
subprocess.call(args) == 0
except Exception as e: # pylint: disable=broad-except
click.secho("Could not install Python packages -> %s" % e,
fg="red",
err=True)
def build_contrib_pysite_package(target_dir, with_metadata=True):
systype = util.get_systype()
if os.path.isdir(target_dir):
fs.rmtree(target_dir)
os.makedirs(target_dir)
# build dependencies
args = [
get_pythonexe_path(),
"-m",
"pip",
"install",
"--no-compile",
"-t",
target_dir,
]
if "linux" in systype:
args.extend(["--no-binary", ":all:"])
for dep in get_contrib_pysite_deps():
subprocess.check_call(args + [dep])
# build manifests
with open(os.path.join(target_dir, "package.json"), "w") as fp:
json.dump(
dict(
name="contrib-pysite",
version="2.%d%d.%s"
% (
sys.version_info.major,
sys.version_info.minor,
date.today().strftime("%y%m%d"),
),
system=list(
set([systype, "linux_armv6l", "linux_armv7l", "linux_armv8l"])
)
if systype.startswith("linux_arm")
else systype,
description="Extra Python package for PlatformIO Core",
keywords=["platformio", "platformio-core"],
homepage="https://docs.platformio.org/page/core/index.html",
repository={
"type": "git",
"url": "https://github.com/platformio/platformio-core",
},
),
fp,
)
# generate package metadata
if with_metadata:
pm = ToolPackageManager()
pkg = PackageItem(target_dir)
pkg.metadata = pm.build_metadata(
target_dir, PackageSpec(owner="platformio", name="contrib-pysite")
)
pkg.dump_meta()
# remove unused files
for root, dirs, files in os.walk(target_dir):
for t in ("_test", "test", "tests"):
if t in dirs:
shutil.rmtree(os.path.join(root, t))
for name in files:
if name.endswith((".chm", ".pyc")):
os.remove(os.path.join(root, name))
return target_dir
