def test_package_id_lt():
"""Test PackageId.__lt__"""
package_id_1 = PackageId(PackageType.PROTOCOL,
PublicId("author", "name", "0.1.0"))
package_id_2 = PackageId(PackageType.PROTOCOL,
PublicId("author", "name", "0.2.0"))
assert package_id_1 < package_id_2
def test_package_can_not_upgraded_cause_required(self):
"""Test no package in registry."""
with self.with_config_update():
with patch(
"aea.cli.upgrade.ItemRemoveHelper.check_remove",
return_value=(
set([
PackageId("connection",
PublicId("test", "test", "0.0.1"))
]),
set(),
dict(),
),
), pytest.raises(
ClickException,
match=r"Can not upgrade .* because it is required by '.*'",
):
self.runner.invoke(
cli,
[
"upgrade",
*self.LOCAL,
self.ITEM_TYPE,
f"{self.ITEM_PUBLIC_ID.author}/{self.ITEM_PUBLIC_ID.name}:latest",
],
standalone_mode=False,
catch_exceptions=False,
)
def check_remove(
self, item_type: str, item_public_id: PublicId
) -> Tuple[Set[PackageId], Set[PackageId], Dict[PackageId, Set[PackageId]]]:
"""
Check item can be removed from agent.
required by - set of components that requires this component
can be deleted - set of dependencies used only by component so can be deleted
can not be deleted - dict - keys - packages can not be deleted, values are set of packages requireed by.
:return: Tuple[required by, can be deleted, can not be deleted.]
"""
package_id = PackageId(item_type, item_public_id)
item = self.get_item_config(package_id)
agent_deps = self.get_agent_dependencies_with_reverse_dependencies()
item_deps = self.get_item_dependencies_with_reverse_dependencies(
item, package_id
)
can_be_removed = set()
can_not_be_removed = dict()
for dep_key, deps in item_deps.items():
if agent_deps[dep_key] == deps:
can_be_removed.add(dep_key)
else:
can_not_be_removed[dep_key] = agent_deps[dep_key] - deps
return agent_deps[package_id], can_be_removed, can_not_be_removed
def test_agent_config_package_dependencies():
"""Test agent config package dependencies."""
agent_config = AgentConfig("name", "author")
assert agent_config.package_dependencies == set()
pid = PublicId("author", "name", "0.1.0")
agent_config.protocols.add(pid)
agent_config.connections.add(pid)
agent_config.contracts.add(pid)
agent_config.skills.add(pid)
assert agent_config.package_dependencies == {
PackageId(PackageType.PROTOCOL, pid),
PackageId(PackageType.CONNECTION, pid),
PackageId(PackageType.CONTRACT, pid),
PackageId(PackageType.SKILL, pid),
}
def test_component_configuration_removed_from_agent_config(self):
"""Test component configuration removed from agent config."""
with cd(self._get_cwd()):
self.run_cli_command(
"add", "--local", self.ITEM_TYPE, str(self.ITEM_PUBLIC_ID)
)
self.run_cli_command("add", "--local", "connection", "fetchai/http_server")
self.runner.invoke(
cli,
[
"config",
"set",
"vendor.fetchai.connections.soef.config.api_key",
"some_api_key",
],
standalone_mode=False,
catch_exceptions=False,
)
self.runner.invoke(
cli,
[
"config",
"set",
"vendor.fetchai.connections.http_server.config.port",
"9000",
],
standalone_mode=False,
catch_exceptions=False,
)
config = self.load_config()
assert config.component_configurations
assert (
PackageId(self.ITEM_TYPE, self.ITEM_PUBLIC_ID)
in config.component_configurations
)
self.run_cli_command("remove", self.ITEM_TYPE, str(self.ITEM_PUBLIC_ID))
config = self.load_config()
assert (
PackageId(self.ITEM_TYPE, self.ITEM_PUBLIC_ID)
not in config.component_configurations
)
assert config.component_configurations
def test_package_can_not_be_removed_cause_required_by_another_package(
self):
"""Test package (oef_search) can not be removed cause required by another package (soef)."""
required_by, can_be_removed, can_not_be_removed = self.check_remove(
self.DEPENDENCY_TYPE, self.DEPENDENCY_PUBLIC_ID)
assert PackageId(self.ITEM_TYPE, self.ITEM_PUBLIC_ID) in required_by
assert not can_be_removed
assert not can_not_be_removed
def find_all_packages_ids() -> Set[PackageId]:
"""Find all packages ids."""
package_ids: Set[PackageId] = set()
for configuration_file in find_all_configuration_files():
package_type = PackageType(configuration_file.parts[-3][:-1])
package_public_id = get_public_id_from_yaml(configuration_file)
package_id = PackageId(package_type, package_public_id)
package_ids.add(package_id)
return package_ids
def get_public_ids_to_update() -> Set[PackageId]:
"""
Get all the public ids to be updated.
In particular, a package DOES NOT NEED a version bump if:
- the package is a "scaffold" package;
- the package is no longer present
- the package hasn't change since the last release;
- the public ids of the local package and the package in the registry
are already the same.
"""
result: Set[PackageId] = set()
last = get_hashes_from_last_release()
now = get_hashes_from_current_release()
last_by_type = split_hashes_by_type(last)
now_by_type = split_hashes_by_type(now)
for type_ in TYPES:
for key, value in last_by_type[type_].items():
# if the package is a "scaffold" package, skip;
if key == "scaffold":
print("Package `{}` of type `{}` is never bumped!".format(
key, type_))
continue
# if the package is no longer present, skip;
if key not in now_by_type[type_]:
print("Package `{}` of type `{}` no longer present!".format(
key, type_))
continue
# if the package hasn't change since the last release, skip;
if now_by_type[type_][key] == value:
print(
"Package `{}` of type `{}` has not changed since last release!"
.format(key, type_))
continue
# load public id in the registry if any
name = key
configuration_file_path = get_configuration_file_path(type_, name)
current_public_id = get_public_id_from_yaml(
configuration_file_path)
deployed_public_id = public_id_in_registry(type_, name)
difference = minor_version_difference(current_public_id,
deployed_public_id)
# check if the public ids of the local package and the package in the registry are already the same.
package_info = f"Package `{name}` of type `{type_}`"
public_id_info = f"current id `{current_public_id}` and deployed id `{deployed_public_id}`"
if difference == 0:
print(f"{package_info} needs to be bumped!")
result.add(PackageId(type_[:-1], current_public_id))
elif difference == 1:
print(f"{package_info} already at correct version!")
continue
else:
print(f"{package_info} has {public_id_info}. Error!")
sys.exit(1)
return result
def find_all_packages_ids() -> Set[PackageId]:
"""Find all packages ids."""
package_ids: Set[PackageId] = set()
packages_dir = Path("packages")
for configuration_file in packages_dir.glob("*/*/*/*.yaml"):
package_type = PackageType(configuration_file.parts[2][:-1])
package_public_id = get_public_id_from_yaml(configuration_file)
package_id = PackageId(package_type, package_public_id)
package_ids.add(package_id)
return package_ids
def _get_item_requirements(
item: PackageConfiguration,
) -> Generator[PackageId, None, None]:
"""
List all the requiemenents for item provided.
:return: generator with package ids: (type, public_id)
"""
for item_type in map(str, ComponentType):
items = getattr(item, f"{item_type}s", set())
for item_public_id in items:
yield PackageId(item_type, item_public_id)
def setup(cls):
"""Set the test up."""
super(TestRemoveAndDependencies, cls).setup()
cls.DEPENDENCY_PACKAGE_ID = PackageId(
cls.DEPENDENCY_TYPE, cls.DEPENDENCY_PUBLIC_ID
)
result = cls.runner.invoke(
cli,
["-v", "DEBUG", "add", "--local", cls.ITEM_TYPE, str(cls.ITEM_PUBLIC_ID)],
standalone_mode=False,
)
assert result.exit_code == 0
def get_all_package_ids() -> Set[PackageId]:
"""Get all the package ids in the local repository."""
result: Set[PackageId] = set()
now = get_hashes_from_current_release()
now_by_type = split_hashes_by_type(now)
for type_, name_to_hashes in now_by_type.items():
for name, _ in name_to_hashes.items():
if name in TEST_PROTOCOLS:
continue
configuration_file_path = get_configuration_file_path(type_, name)
public_id = get_public_id_from_yaml(configuration_file_path)
package_id = PackageId(PackageType(type_[:-1]), public_id)
result.add(package_id)
return result
def find_all_packages_ids() -> Set[PackageId]:
"""Find all packages ids."""
package_ids: Set[PackageId] = set()
packages_dir = Path("packages")
config_files = [
path for path in packages_dir.glob("*/*/*/*.yaml")
if any([file in str(path) for file in CONFIG_FILE_NAMES])
]
for configuration_file in chain(config_files, default_config_file_paths()):
package_type = PackageType(configuration_file.parts[-3][:-1])
package_public_id = get_public_id_from_yaml(configuration_file)
package_id = PackageId(package_type, package_public_id)
package_ids.add(package_id)
return package_ids
def _get_item_requirements(self,
item: PackageConfiguration,
ignore_non_vendor: bool = False
) -> Generator[PackageId, None, None]:
"""
List all the requirements for item provided.
:return: generator with package ids: (type, public_id)
"""
for item_type in map(str, ComponentType):
items = getattr(item, f"{item_type}s", set())
for item_public_id in items:
if ignore_non_vendor and is_item_present(
self._ctx.cwd,
self._ctx.agent_config,
item_type,
item_public_id,
is_vendor=False,
):
continue
yield PackageId(item_type, item_public_id)
def setup(self):
"""Set the test up."""
self.runner = CliRunner()
self.agent_name = "myagent"
self.cwd = os.getcwd()
self.t = tempfile.mkdtemp()
# copy the 'packages' directory in the parent of the agent folder.
shutil.copytree(Path(CUR_PATH, "..", "packages"),
Path(self.t, "packages"))
os.chdir(self.t)
result = self.runner.invoke(
cli,
[*CLI_LOG_OPTION, "init", "--local", "--author", AUTHOR],
standalone_mode=False,
)
assert result.exit_code == 0
result = self.runner.invoke(
cli,
[*CLI_LOG_OPTION, "create", "--local", self.agent_name],
standalone_mode=False,
)
assert result.exit_code == 0
os.chdir(self.agent_name)
# add connection first time
self.DEPENDENCY_PACKAGE_ID = PackageId(self.DEPENDENCY_TYPE,
self.DEPENDENCY_PUBLIC_ID)
result = self.runner.invoke(
cli,
[
"-v", "DEBUG", "add", "--local", self.ITEM_TYPE,
str(self.ITEM_PUBLIC_ID)
],
standalone_mode=False,
catch_exceptions=False,
)
def extract_package_id(match: Match) -> PackageId:
package_public_id = match.group(1)
package_id = PackageId(PackageType.AGENT,
PublicId.from_str(package_public_id))
return package_id
def extract_package_id(match: Match) -> PackageId:
package_type, package = match.group(1), match.group(2)
package_id = PackageId(PackageType(package_type),
PublicId.from_str(package))
return package_id
def _get_item_folder(self) -> Path:
"""Get item package folder."""
return Path(self.cwd) / ItemRemoveHelper.get_component_directory(
PackageId(self.item_type, self.item_id)
)
def is_non_vendor(self) -> bool:
"""Check is package specified is non vendor."""
path = ItemRemoveHelper.get_component_directory(
PackageId(self.item_type, self.item_public_id)
)
return "vendor" not in Path(path).parts[:2]
def _eject_item(
ctx: Context,
item_type: str,
public_id: PublicId,
quiet: bool = True,
with_symlinks: bool = False,
) -> None:
"""
Eject item from installed (vendor) to custom folder.
:param ctx: context object.
:param item_type: item type.
:param public_id: item public ID.
:param quiet: if false, the function will ask the user in case of recursive eject.
:return: None
:raises: ClickException if item is absent at source path or present at destenation path.
"""
# we know cli_author is set because of the above checks.
cli_author: str = cast(str, ctx.config.get("cli_author"))
item_type_plural = item_type + "s"
if not is_item_present(
ctx.cwd,
ctx.agent_config,
item_type,
public_id,
is_vendor=True,
with_version=True,
): # pragma: no cover
raise click.ClickException(
f"{item_type.title()} {public_id} not found in agent's vendor items."
)
src = get_package_path(ctx.cwd, item_type, public_id)
dst = get_package_path(ctx.cwd, item_type, public_id, is_vendor=False)
if is_item_present(ctx.cwd,
ctx.agent_config,
item_type,
public_id,
is_vendor=False): # pragma: no cover
raise click.ClickException(
f"{item_type.title()} {public_id} is already a non-vendor package."
)
configuration = load_item_config(item_type, Path(src))
if public_id.package_version.is_latest:
# get 'concrete' public id, in case it is 'latest'
component_prefix = ComponentType(
item_type), public_id.author, public_id.name
component_id = get_latest_component_id_from_prefix(
ctx.agent_config, component_prefix)
# component id is necessarily found, due to the checks above.
public_id = cast(ComponentId, component_id).public_id
package_id = PackageId(PackageType(item_type), public_id)
click.echo(
f"Ejecting item {package_id.package_type.value} {str(package_id.public_id)}"
)
# first, eject all the vendor packages that depend on this
item_remover = ItemRemoveHelper(ctx, ignore_non_vendor=True)
reverse_dependencies = (
item_remover.get_agent_dependencies_with_reverse_dependencies())
reverse_reachable_dependencies = reachable_nodes(reverse_dependencies,
{package_id})
# the reversed topological order of a graph
# is the topological order of the reverse graph.
eject_order = list(
reversed(find_topological_order(reverse_reachable_dependencies)))
eject_order.remove(package_id)
if len(eject_order) > 0 and not quiet:
click.echo(
f"The following vendor packages will be ejected: {eject_order}")
answer = click.confirm("Do you want to proceed?")
if not answer:
click.echo("Aborted.")
return
for dependency_package_id in eject_order:
# 'dependency_package_id' depends on 'package_id',
# so we need to eject it first
_eject_item(
ctx,
dependency_package_id.package_type.value,
dependency_package_id.public_id,
quiet=True,
)
# copy the vendor package into the non-vendor packages
ctx.clean_paths.append(dst)
copy_package_directory(Path(src), dst)
new_public_id = PublicId(cli_author, public_id.name, DEFAULT_VERSION)
current_version = Version(aea.__version__)
new_aea_range = (
configuration.aea_version
if configuration.aea_version_specifiers.contains(current_version) else
compute_specifier_from_version(current_version))
update_item_config(
item_type,
Path(dst),
author=new_public_id.author,
version=new_public_id.version,
aea_version=new_aea_range,
)
update_item_public_id_in_init(item_type, Path(dst), new_public_id)
shutil.rmtree(src)
# update references in all the other packages
component_type = ComponentType(item_type_plural[:-1])
old_component_id = ComponentId(component_type, public_id)
new_component_id = ComponentId(component_type, new_public_id)
update_references(ctx, {old_component_id: new_component_id})
# need to reload agent configuration with the updated references
try_to_load_agent_config(ctx)
# replace import statements in all the non-vendor packages
replace_all_import_statements(Path(ctx.cwd), ComponentType(item_type),
public_id, new_public_id)
# fingerprint all (non-vendor) packages
fingerprint_all(ctx)
if with_symlinks:
click.echo(
"Adding symlinks from vendor to non-vendor and packages to vendor folders."
)
create_symlink_vendor_to_local(ctx, item_type, new_public_id)
create_symlink_packages_to_vendor(ctx)
click.echo(
f"Successfully ejected {item_type} {public_id} to {dst} as {new_public_id}."
)
def test_package_id_to_uri_path():
"""Test PackageId.to_uri_path"""
package_id = PackageId(PackageType.PROTOCOL,
PublicId("author", "name", "0.1.0"))
assert package_id.to_uri_path == "protocol/author/name/0.1.0"
def _add_package_type(package_type, public_id_str):
return PackageId(package_type, PublicId.from_str(public_id_str))
def _add_package_type(package_type: PackageType,
public_id_str: str) -> PackageId:
return PackageId(package_type, PublicId.from_str(public_id_str))
def test_package_id_version():
"""Test PackageId.version"""
package_id = PackageId(PackageType.PROTOCOL,
PublicId("author", "name", "0.1.0"))
assert package_id.version == "0.1.0"
def test_package_id_str():
"""Test PackageId.__str__"""
package_id = PackageId(PackageType.PROTOCOL,
PublicId("author", "name", "0.1.0"))
assert str(package_id) == "(protocol, author/name:0.1.0)"
