def _update(self, pkg, outdated, silent=False):
if pkg.metadata.spec.external:
vcs = VCSClientFactory.new(pkg.path, pkg.metadata.spec.url)
assert vcs.update()
pkg.metadata.version = self._fetch_vcs_latest_version(pkg)
pkg.dump_meta()
return pkg
new_pkg = self.install(
PackageSpec(
id=pkg.metadata.spec.id,
owner=pkg.metadata.spec.owner,
name=pkg.metadata.spec.name,
requirements=outdated.wanted or outdated.latest,
),
silent=silent,
)
if new_pkg:
old_pkg = self.get_package(
PackageSpec(
id=pkg.metadata.spec.id,
owner=pkg.metadata.spec.owner,
name=pkg.metadata.name,
requirements=pkg.metadata.version,
)
)
if old_pkg:
self.uninstall(old_pkg, silent=silent, skip_dependencies=True)
return new_pkg
def test_spec_as_dict():
assert not jsondiff.diff(
PackageSpec("bob/[email protected]").as_dict(),
{
"owner": "bob",
"id": None,
"name": "foo",
"requirements": "1.2.3",
"url": None,
},
)
assert not jsondiff.diff(
PackageSpec(
"https://github.com/platformio/platformio-core/archive/develop.zip?param=value"
" @ !=2").as_dict(),
{
"owner":
None,
"id":
None,
"name":
"platformio-core",
"requirements":
"!=2",
"url":
"https://github.com/platformio/platformio-core/archive/develop.zip?param=value",
},
)
def test_metadata_load(tmpdir_factory):
contents = """
{
"name": "foo",
"spec": {
"name": "foo",
"owner": "username",
"requirements": "!=3.4.5"
},
"type": "platform",
"version": "0.1.3"
}
"""
pkg_dir = tmpdir_factory.mktemp("package")
dst = pkg_dir.join(".piopm")
dst.write(contents)
metadata = PackageMetaData.load(str(dst))
assert metadata.version == semantic_version.Version("0.1.3")
assert metadata == PackageMetaData(
PackageType.PLATFORM,
"foo",
"0.1.3",
spec=PackageSpec(owner="username", name="foo", requirements="!=3.4.5"),
)
piopm_path = pkg_dir.join(".piopm")
metadata = PackageMetaData(PackageType.LIBRARY,
"mylib",
version="1.2.3",
spec=PackageSpec("mylib"))
metadata.dump(str(piopm_path))
restored_metadata = PackageMetaData.load(str(piopm_path))
assert metadata == restored_metadata
def test_update_with_metadata(isolated_pio_core, tmpdir_factory):
storage_dir = tmpdir_factory.mktemp("storage")
lm = LibraryPackageManager(str(storage_dir))
# test non SemVer in registry
pkg = lm.install("RadioHead @ <1.90", silent=True)
outdated = lm.outdated(pkg)
assert str(outdated.current) == "1.89.0"
assert outdated.latest > semantic_version.Version("1.100.0")
pkg = lm.install("ArduinoJson @ 5.10.1", silent=True)
# tesy latest
outdated = lm.outdated(pkg)
assert str(outdated.current) == "5.10.1"
assert outdated.wanted is None
assert outdated.latest > outdated.current
assert outdated.latest > semantic_version.Version("5.99.99")
# test wanted
outdated = lm.outdated(pkg, PackageSpec("ArduinoJson@~5"))
assert str(outdated.current) == "5.10.1"
assert str(outdated.wanted) == "5.13.4"
assert outdated.latest > semantic_version.Version("6.16.0")
# update to the wanted 5.x
new_pkg = lm.update("ArduinoJson@^5", PackageSpec("ArduinoJson@^5"), silent=True)
assert str(new_pkg.metadata.version) == "5.13.4"
# check that old version is removed
assert len(lm.get_installed()) == 2
# update to the latest
lm = LibraryPackageManager(str(storage_dir))
pkg = lm.update("ArduinoJson", silent=True)
assert pkg.metadata.version == outdated.latest
def test_spec_owner():
assert PackageSpec("alice/foo library") == PackageSpec(owner="alice",
name="foo library")
spec = PackageSpec(" Bob / BarUpper ")
assert spec != PackageSpec(owner="BOB", name="BARUPPER")
assert spec.owner == "Bob"
assert spec.name == "BarUpper"
def get_package_spec(self, name):
version = self.packages[name].get("version", "")
if any(c in version for c in (":", "/", "@")):
return PackageSpec("%s=%s" % (name, version))
return PackageSpec(owner=self.packages[name].get("owner"),
name=name,
requirements=version)
def test_build_legacy_spec(isolated_pio_core, tmpdir_factory):
storage_dir = tmpdir_factory.mktemp("storage")
pm = PlatformPackageManager(str(storage_dir))
# test src manifest
pkg1_dir = storage_dir.join("pkg-1").mkdir()
pkg1_dir.join(".pio").mkdir().join(".piopkgmanager.json").write("""
{
"name": "StreamSpy-0.0.1.tar",
"url": "https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz",
"requirements": null
}
""")
assert pm.build_legacy_spec(str(pkg1_dir)) == PackageSpec(
name="StreamSpy-0.0.1.tar",
url="https://dl.platformio.org/e8936b7/StreamSpy-0.0.1.tar.gz",
)
# without src manifest
pkg2_dir = storage_dir.join("pkg-2").mkdir()
pkg2_dir.join("main.cpp").write("")
with pytest.raises(MissingPackageManifestError):
pm.build_legacy_spec(str(pkg2_dir))
# with package manifest
pkg3_dir = storage_dir.join("pkg-3").mkdir()
pkg3_dir.join("platform.json").write(
'{"name": "pkg3", "version": "1.2.0"}')
assert pm.build_legacy_spec(str(pkg3_dir)) == PackageSpec(name="pkg3")
def test_build_metadata(isolated_pio_core, tmpdir_factory):
pm = PlatformPackageManager()
vcs_revision = "a2ebfd7c0f"
pkg_dir = tmpdir_factory.mktemp("package")
# test package without manifest
with pytest.raises(MissingPackageManifestError):
pm.load_manifest(str(pkg_dir))
with pytest.raises(MissingPackageManifestError):
pm.build_metadata(str(pkg_dir), PackageSpec("MyLib"))
# with manifest
pkg_dir.join("platform.json").write(
'{"name": "Dev-Platform", "version": "1.2.3-alpha.1"}')
metadata = pm.build_metadata(str(pkg_dir),
PackageSpec("owner/platform-name"))
assert metadata.name == "Dev-Platform"
assert str(metadata.version) == "1.2.3-alpha.1"
# with vcs
metadata = pm.build_metadata(str(pkg_dir),
PackageSpec("owner/platform-name"),
vcs_revision)
assert str(metadata.version) == ("1.2.3-alpha.1+sha." + vcs_revision)
assert metadata.version.build[1] == vcs_revision
def _save_deps(ctx, pkgs, action="add"):
specs = []
for library, pkg in pkgs.items():
spec = PackageSpec(library)
if spec.external:
specs.append(spec)
else:
specs.append(
PackageSpec(
owner=pkg.metadata.spec.owner,
name=pkg.metadata.spec.name,
requirements=spec.requirements
or (
("^%s" % pkg.metadata.version)
if not pkg.metadata.version.build
else pkg.metadata.version
),
)
)
input_dirs = ctx.meta.get(CTX_META_INPUT_DIRS_KEY, [])
project_environments = ctx.meta[CTX_META_PROJECT_ENVIRONMENTS_KEY]
for input_dir in input_dirs:
if not is_platformio_project(input_dir):
continue
save_project_libdeps(input_dir, specs, project_environments, action=action)
def lib_update( # pylint: disable=too-many-arguments
ctx, libraries, only_check, dry_run, silent, json_output
):
storage_dirs = ctx.meta[CTX_META_STORAGE_DIRS_KEY]
only_check = dry_run or only_check
json_result = {}
for storage_dir in storage_dirs:
if not json_output:
print_storage_header(storage_dirs, storage_dir)
lib_deps = ctx.meta.get(CTX_META_STORAGE_LIBDEPS_KEY, {}).get(storage_dir, [])
lm = LibraryPackageManager(storage_dir)
_libraries = libraries or lib_deps or lm.get_installed()
if only_check and json_output:
result = []
for library in _libraries:
spec = None
pkg = None
if isinstance(library, PackageItem):
pkg = library
else:
spec = PackageSpec(library)
pkg = lm.get_package(spec)
if not pkg:
continue
outdated = lm.outdated(pkg, spec)
if not outdated.is_outdated(allow_incompatible=True):
continue
manifest = lm.legacy_load_manifest(pkg)
manifest["versionWanted"] = (
str(outdated.wanted) if outdated.wanted else None
)
manifest["versionLatest"] = (
str(outdated.latest) if outdated.latest else None
)
result.append(manifest)
json_result[storage_dir] = result
else:
for library in _libraries:
to_spec = (
None if isinstance(library, PackageItem) else PackageSpec(library)
)
try:
lm.update(
library, to_spec=to_spec, only_check=only_check, silent=silent
)
except UnknownPackageError as e:
if library not in lib_deps:
raise e
if json_output:
return click.echo(
dump_json_to_unicode(
json_result[storage_dirs[0]] if len(storage_dirs) == 1 else json_result
)
)
return True
def test_spec_as_dependency():
assert PackageSpec("owner/pkgname").as_dependency() == "owner/pkgname"
assert PackageSpec(owner="owner",
name="pkgname").as_dependency() == "owner/pkgname"
assert PackageSpec("bob/foo @ ^1.2.3").as_dependency() == "bob/foo@^1.2.3"
assert (
PackageSpec("https://github.com/o/r/a/develop.zip?param=value @ !=2"
).as_dependency() ==
"https://github.com/o/r/a/develop.zip?param=value @ !=2")
assert (
PackageSpec("wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/"
).as_dependency() ==
"wolfSSL=https://os.mbed.com/users/wolfSSL/code/wolfSSL/")
def test_spec_local_urls(tmpdir_factory):
assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec(
url="file:///tmp/foo.tar.gz", name="foo")
assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec(
url="file:///tmp/bar.zip", name="customName")
assert PackageSpec("file:///tmp/some-lib/") == PackageSpec(
url="file:///tmp/some-lib/", name="some-lib")
assert PackageSpec("file:///tmp/foo.tar.gz@~2.3.0-beta.1") == PackageSpec(
url="file:///tmp/foo.tar.gz", name="foo", requirements="~2.3.0-beta.1")
# detached folder with "@" symbol
pkg_dir = tmpdir_factory.mktemp("storage").join("[email protected]").mkdir()
assert PackageSpec("file://%s" % str(pkg_dir)) == PackageSpec(
name="detached", url="file://%s" % pkg_dir)
def test_spec_external_urls():
assert PackageSpec(
"https://github.com/platformio/platformio-core/archive/develop.zip"
) == PackageSpec(
url="https://github.com/platformio/platformio-core/archive/develop.zip",
name="platformio-core",
)
assert PackageSpec(
"https://github.com/platformio/platformio-core/archive/develop.zip?param=value"
" @ !=2") == PackageSpec(
url="https://github.com/platformio/platformio-core/archive/"
"develop.zip?param=value",
name="platformio-core",
requirements="!=2",
)
spec = PackageSpec(
"Custom-Name="
"https://github.com/platformio/platformio-core/archive/[email protected]"
)
assert spec.external
assert spec.has_custom_name()
assert spec.name == "Custom-Name"
assert spec == PackageSpec(
url=
"https://github.com/platformio/platformio-core/archive/develop.tar.gz",
name="Custom-Name",
requirements="4.4.0",
)
def test_spec_local_urls(tmpdir_factory):
assert PackageSpec("file:///tmp/foo.tar.gz") == PackageSpec(
url="file:///tmp/foo.tar.gz", name="foo")
assert PackageSpec("customName=file:///tmp/bar.zip") == PackageSpec(
url="file:///tmp/bar.zip", name="customName")
assert PackageSpec("file:///tmp/some-lib/") == PackageSpec(
url="file:///tmp/some-lib/", name="some-lib")
# detached package
assert PackageSpec(
"file:///tmp/some-lib@src-67e1043a673d2") == PackageSpec(
url="file:///tmp/some-lib@src-67e1043a673d2", name="some-lib")
# detached folder without scheme
pkg_dir = tmpdir_factory.mktemp("storage").join("[email protected]").mkdir()
assert PackageSpec(str(pkg_dir)) == PackageSpec(name="detached",
url="file://%s" % pkg_dir)
def test_find_pkg_root(isolated_pio_core, tmpdir_factory):
# has manifest
pkg_dir = tmpdir_factory.mktemp("package-has-manifest")
root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir()
root_dir.join("platform.json").write("")
pm = PlatformPackageManager()
found_dir = pm.find_pkg_root(str(pkg_dir), spec=None)
assert os.path.realpath(str(root_dir)) == os.path.realpath(found_dir)
# does not have manifest
pkg_dir = tmpdir_factory.mktemp("package-does-not-have-manifest")
pkg_dir.join("nested").mkdir().join("folder").mkdir().join(
"readme.txt").write("")
pm = PlatformPackageManager()
with pytest.raises(MissingPackageManifestError):
pm.find_pkg_root(str(pkg_dir), spec=None)
# library package without manifest, should find source root
pkg_dir = tmpdir_factory.mktemp("library-package-without-manifest")
root_dir = pkg_dir.join("nested").mkdir().join("folder").mkdir()
root_dir.join("src").mkdir().join("main.cpp").write("")
root_dir.join("include").mkdir().join("main.h").write("")
assert os.path.realpath(str(root_dir)) == os.path.realpath(
LibraryPackageManager.find_library_root(str(pkg_dir)))
# library manager should create "library.json"
lm = LibraryPackageManager()
spec = PackageSpec("[email protected]")
pkg_root = lm.find_pkg_root(str(pkg_dir), spec)
manifest_path = os.path.join(pkg_root, "library.json")
assert os.path.realpath(str(root_dir)) == os.path.realpath(pkg_root)
assert os.path.isfile(manifest_path)
manifest = lm.load_manifest(pkg_root)
assert manifest["name"] == "custom-name"
assert "0.0.0" in str(manifest["version"])
def remove_unnecessary_core_packages(dry_run=False):
candidates = []
pm = ToolPackageManager()
best_pkg_versions = {}
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
best_pkg_versions[pkg.metadata.name] = pkg.metadata.version
for pkg in pm.get_installed():
skip_conds = [
os.path.isfile(os.path.join(pkg.path, ".piokeep")),
pkg.metadata.spec.owner != "platformio",
pkg.metadata.name not in best_pkg_versions,
pkg.metadata.name in best_pkg_versions
and pkg.metadata.version == best_pkg_versions[pkg.metadata.name],
]
if not any(skip_conds):
candidates.append(pkg)
if dry_run:
return candidates
for pkg in candidates:
pm.uninstall(pkg)
return candidates
def _update_pkg_metadata(_):
pm = ToolPackageManager()
for pkg in pm.get_installed():
if not pkg.metadata or pkg.metadata.spec.external or pkg.metadata.spec.id:
continue
result = pm.search_registry_packages(PackageSpec(name=pkg.metadata.name))
if len(result) != 1:
continue
result = result[0]
pkg.metadata.spec = PackageSpec(
id=result["id"],
owner=result["owner"]["username"],
name=result["name"],
)
pkg.dump_meta()
return True
def test_metadata_as_dict():
metadata = PackageMetaData(PackageType.LIBRARY, "foo", "1.2.3")
# test setter
metadata.version = "0.1.2+12345"
assert metadata.version == semantic_version.Version("0.1.2+12345")
assert not jsondiff.diff(
metadata.as_dict(),
{
"type": PackageType.LIBRARY,
"name": "foo",
"version": "0.1.2+12345",
"spec": None,
},
)
assert not jsondiff.diff(
PackageMetaData(
PackageType.TOOL,
"toolchain",
"2.0.5",
PackageSpec("platformio/toolchain@~2.0.0"),
).as_dict(),
{
"type": PackageType.TOOL,
"name": "toolchain",
"version": "2.0.5",
"spec": {
"owner": "platformio",
"id": None,
"name": "toolchain",
"requirements": "~2.0.0",
"url": None,
},
},
)
def test_install_from_url(isolated_pio_core, tmpdir_factory):
tmp_dir = tmpdir_factory.mktemp("tmp")
storage_dir = tmpdir_factory.mktemp("storage")
lm = LibraryPackageManager(str(storage_dir))
# install from local directory
src_dir = tmp_dir.join("local-lib-dir").mkdir()
src_dir.join("main.cpp").write("")
spec = PackageSpec("file://%s" % src_dir)
pkg = lm.install(spec, silent=True)
assert os.path.isfile(os.path.join(pkg.path, "main.cpp"))
manifest = lm.load_manifest(pkg)
assert manifest["name"] == "local-lib-dir"
assert manifest["version"].startswith("0.0.0+")
assert spec == pkg.metadata.spec
# install from local archive
src_dir = tmp_dir.join("archive-src").mkdir()
root_dir = src_dir.mkdir("root")
root_dir.mkdir("src").join("main.cpp").write("#include <stdio.h>")
root_dir.join("library.json").write(
'{"name": "manifest-lib-name", "version": "2.0.0"}'
)
tarball_path = PackagePacker(str(src_dir)).pack(str(tmp_dir))
spec = PackageSpec("file://%s" % tarball_path)
pkg = lm.install(spec, silent=True)
assert os.path.isfile(os.path.join(pkg.path, "src", "main.cpp"))
assert pkg == lm.get_package(spec)
assert spec == pkg.metadata.spec
# install from registry
src_dir = tmp_dir.join("registry-1").mkdir()
src_dir.join("library.properties").write(
"""
name = wifilib
version = 5.2.7
"""
)
spec = PackageSpec("company/wifilib @ ^5")
pkg = lm.install_from_url("file://%s" % src_dir, spec)
assert str(pkg.metadata.version) == "5.2.7"
# check package folder names
lm.memcache_reset()
assert ["local-lib-dir", "manifest-lib-name", "wifilib"] == [
os.path.basename(pkg.path) for pkg in lm.get_installed()
]
def get_all_known_libs():
known_libs = []
for feature in FEATURE_CONFIG:
feat = FEATURE_CONFIG[feature]
if not 'lib_deps' in feat:
continue
for dep in feat['lib_deps']:
known_libs.append(PackageSpec(dep).name)
return known_libs
def get_installed_core_packages():
result = []
pm = ToolPackageManager()
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if pkg:
result.append(pkg)
return result
def package_unpublish(package, type, undo): # pylint: disable=redefined-builtin
spec = PackageSpec(package)
response = RegistryClient().unpublish_package(
type=type,
name=spec.name,
owner=spec.owner,
version=str(spec.requirements),
undo=undo,
)
click.secho(response.get("message"), fg="green")
def _install_dependency(self, dependency, silent=False):
if set(["name", "version"]) <= set(dependency.keys()) and any(
c in dependency["version"] for c in (":", "/", "@")):
spec = PackageSpec("%s=%s" %
(dependency["name"], dependency["version"]))
else:
spec = PackageSpec(
owner=dependency.get("owner"),
name=dependency.get("name"),
requirements=dependency.get("version"),
)
search_filters = {
key: value
for key, value in dependency.items()
if key in ("authors", "platforms", "frameworks")
}
return self._install(spec,
search_filters=search_filters or None,
silent=silent)
def _install_dependency(self, dependency, silent=False):
spec = PackageSpec(name=dependency.get("name"),
requirements=dependency.get("version"))
search_filters = {
key: value
for key, value in dependency.items()
if key in ("authors", "platforms", "frameworks")
}
return self._install(spec,
search_filters=search_filters or None,
silent=silent)
def update_core_packages(only_check=False, silent=False):
pm = ToolPackageManager()
for name, requirements in __core_packages__.items():
spec = PackageSpec(owner="platformio", name=name, requirements=requirements)
pkg = pm.get_package(spec)
if not pkg:
continue
if not silent or pm.outdated(pkg, spec).is_outdated():
pm.update(pkg, spec, only_check=only_check)
if not only_check:
remove_unnecessary_core_packages()
return True
def build_legacy_spec(self, pkg_dir):
# find src manifest
src_manifest_name = ".piopkgmanager.json"
src_manifest_path = None
for name in os.listdir(pkg_dir):
if not os.path.isfile(os.path.join(pkg_dir, name, src_manifest_name)):
continue
src_manifest_path = os.path.join(pkg_dir, name, src_manifest_name)
break
if src_manifest_path:
src_manifest = fs.load_json(src_manifest_path)
return PackageSpec(
name=src_manifest.get("name"),
url=src_manifest.get("url"),
requirements=src_manifest.get("requirements"),
)
# fall back to a package manifest
manifest = self.load_manifest(pkg_dir)
return PackageSpec(name=manifest.get("name"))
def get_core_package_dir(name):
if name not in __core_packages__:
raise exception.PlatformioException("Please upgrade PlatformIO Core")
pm = ToolPackageManager()
spec = PackageSpec(owner="platformio",
name=name,
requirements=__core_packages__[name])
pkg = pm.get_package(spec)
if pkg:
return pkg.path
assert pm.install(spec)
_remove_unnecessary_packages()
return pm.get_package(spec).path
def test_metadata_dump(tmpdir_factory):
pkg_dir = tmpdir_factory.mktemp("package")
metadata = PackageMetaData(
PackageType.TOOL,
"toolchain",
"2.0.5",
PackageSpec("platformio/toolchain@~2.0.0"),
)
dst = pkg_dir.join(".piopm")
metadata.dump(str(dst))
assert os.path.isfile(str(dst))
contents = dst.read()
assert all(s in contents for s in ("null", '"~2.0.0"'))
def uninstall_dependencies(self, pkg, silent=False):
assert isinstance(pkg, PackageItem)
manifest = self.load_manifest(pkg)
if not manifest.get("dependencies"):
return
if not silent:
self.print_message("Removing dependencies...", fg="yellow")
for dependency in manifest.get("dependencies"):
pkg = self.get_package(
PackageSpec(name=dependency.get("name"),
requirements=dependency.get("version")))
if not pkg:
continue
self._uninstall(pkg, silent=silent)
def platform_update( # pylint: disable=too-many-locals, too-many-arguments
platforms, only_packages, only_check, dry_run, silent, json_output):
pm = PlatformPackageManager()
platforms = platforms or pm.get_installed()
only_check = dry_run or only_check
if only_check and json_output:
result = []
for platform in platforms:
spec = None
pkg = None
if isinstance(platform, PackageItem):
pkg = platform
else:
spec = PackageSpec(platform)
pkg = pm.get_package(spec)
if not pkg:
continue
outdated = pm.outdated(pkg, spec)
if (not outdated.is_outdated(allow_incompatible=True)
and not PlatformFactory.new(pkg).are_outdated_packages()):
continue
data = _get_installed_platform_data(pkg,
with_boards=False,
expose_packages=False)
if outdated.is_outdated(allow_incompatible=True):
data["versionLatest"] = (str(outdated.latest)
if outdated.latest else None)
result.append(data)
return click.echo(dump_json_to_unicode(result))
# cleanup cached board and platform lists
cleanup_content_cache("http")
for platform in platforms:
click.echo("Platform %s" % click.style(
platform.metadata.name
if isinstance(platform, PackageItem) else platform,
fg="cyan",
))
click.echo("--------")
pm.update(platform,
only_packages=only_packages,
only_check=only_check,
silent=silent)
click.echo()
return True