def test_package_id_lt(): """Test PackageId.__lt__""" package_id_1 = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.1.0")) package_id_2 = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.2.0")) assert package_id_1 < package_id_2
def test_package_id_from_uri_path_negative(): """Test PackageId.from_uri_path with invalid type""" with pytest.raises( ValueError, match= "Input 'not_a_valid_type/author/package_name/0.1.0' is not well formatted.", ): PackageId.from_uri_path("not_a_valid_type/author/package_name/0.1.0")
def test_package_id_from_uri_path(): """Test PackageId.from_uri_path""" result = PackageId.from_uri_path("skill/author/package_name/0.1.0") assert str(result.package_type) == "skill" assert result.public_id.name == "package_name" assert result.public_id.author == "author" assert result.public_id.version == "0.1.0"
def _get_public_ids_from_uri( uri: URI, ) -> Tuple[Optional[PublicId], Optional[PublicId]]: """ Try get skill and connection id from uri. :param uri: the uri :return: (skill_id if present in uri, connection if present in uri) """ skill_id = None connection_id = None try: package_id = PackageId.from_uri_path(uri.path) package_type = str(package_id.package_type) if package_type == "skill": skill_id = package_id.public_id elif package_type == "connection": connection_id = package_id.public_id else: raise ValueError( f"Invalid package type {package_type} in uri for envelope context." ) except ValueError as e: _default_logger.debug( f"URI - {uri.path} - not a valid package_id id. Error: {e}") return (skill_id, connection_id)
def check_remove( self, item_type: str, item_public_id: PublicId ) -> Tuple[Set[PackageId], Set[PackageId], Dict[PackageId, Set[PackageId]]]: """ Check item can be removed from agent. required by - set of components that requires this component can be deleted - set of dependencies used only by component so can be deleted can not be deleted - dict - keys - packages can not be deleted, values are set of packages requireed by. :return: Tuple[required by, can be deleted, can not be deleted.] """ package_id = PackageId(item_type, item_public_id) item = self.get_item_config(package_id) agent_deps = self.get_agent_dependencies_with_reverse_dependencies() item_deps = self.get_item_dependencies_with_reverse_dependencies( item, package_id ) can_be_removed = set() can_not_be_removed = dict() for dep_key, deps in item_deps.items(): if agent_deps[dep_key] == deps: can_be_removed.add(dep_key) else: can_not_be_removed[dep_key] = agent_deps[dep_key] - deps return agent_deps[package_id], can_be_removed, can_not_be_removed
def test_package_can_not_upgraded_cause_required(self): """Test no package in registry.""" with self.with_config_update(): with patch( "aea.cli.upgrade.ItemRemoveHelper.check_remove", return_value=( set([ PackageId("connection", PublicId("test", "test", "0.0.1")) ]), set(), dict(), ), ), pytest.raises( ClickException, match=r"Can not upgrade .* because it is required by '.*'", ): self.runner.invoke( cli, [ "upgrade", *self.LOCAL, self.ITEM_TYPE, f"{self.ITEM_PUBLIC_ID.author}/{self.ITEM_PUBLIC_ID.name}:latest", ], standalone_mode=False, catch_exceptions=False, )
def test_agent_config_package_dependencies(): """Test agent config package dependencies.""" agent_config = AgentConfig("name", "author") assert agent_config.package_dependencies == set() pid = PublicId("author", "name", "0.1.0") agent_config.protocols.add(pid) agent_config.connections.add(pid) agent_config.contracts.add(pid) agent_config.skills.add(pid) assert agent_config.package_dependencies == { PackageId(PackageType.PROTOCOL, pid), PackageId(PackageType.CONNECTION, pid), PackageId(PackageType.CONTRACT, pid), PackageId(PackageType.SKILL, pid), }
def test_component_configuration_removed_from_agent_config(self): """Test component configuration removed from agent config.""" with cd(self._get_cwd()): self.run_cli_command( "add", "--local", self.ITEM_TYPE, str(self.ITEM_PUBLIC_ID) ) self.run_cli_command("add", "--local", "connection", "fetchai/http_server") self.runner.invoke( cli, [ "config", "set", "vendor.fetchai.connections.soef.config.api_key", "some_api_key", ], standalone_mode=False, catch_exceptions=False, ) self.runner.invoke( cli, [ "config", "set", "vendor.fetchai.connections.http_server.config.port", "9000", ], standalone_mode=False, catch_exceptions=False, ) config = self.load_config() assert config.component_configurations assert ( PackageId(self.ITEM_TYPE, self.ITEM_PUBLIC_ID) in config.component_configurations ) self.run_cli_command("remove", self.ITEM_TYPE, str(self.ITEM_PUBLIC_ID)) config = self.load_config() assert ( PackageId(self.ITEM_TYPE, self.ITEM_PUBLIC_ID) not in config.component_configurations ) assert config.component_configurations
def test_package_can_not_be_removed_cause_required_by_another_package( self): """Test package (oef_search) can not be removed cause required by another package (soef).""" required_by, can_be_removed, can_not_be_removed = self.check_remove( self.DEPENDENCY_TYPE, self.DEPENDENCY_PUBLIC_ID) assert PackageId(self.ITEM_TYPE, self.ITEM_PUBLIC_ID) in required_by assert not can_be_removed assert not can_not_be_removed
def find_all_packages_ids() -> Set[PackageId]: """Find all packages ids.""" package_ids: Set[PackageId] = set() for configuration_file in find_all_configuration_files(): package_type = PackageType(configuration_file.parts[-3][:-1]) package_public_id = get_public_id_from_yaml(configuration_file) package_id = PackageId(package_type, package_public_id) package_ids.add(package_id) return package_ids
def get_public_ids_to_update() -> Set[PackageId]: """ Get all the public ids to be updated. In particular, a package DOES NOT NEED a version bump if: - the package is a "scaffold" package; - the package is no longer present - the package hasn't change since the last release; - the public ids of the local package and the package in the registry are already the same. """ result: Set[PackageId] = set() last = get_hashes_from_last_release() now = get_hashes_from_current_release() last_by_type = split_hashes_by_type(last) now_by_type = split_hashes_by_type(now) for type_ in TYPES: for key, value in last_by_type[type_].items(): # if the package is a "scaffold" package, skip; if key == "scaffold": print("Package `{}` of type `{}` is never bumped!".format( key, type_)) continue # if the package is no longer present, skip; if key not in now_by_type[type_]: print("Package `{}` of type `{}` no longer present!".format( key, type_)) continue # if the package hasn't change since the last release, skip; if now_by_type[type_][key] == value: print( "Package `{}` of type `{}` has not changed since last release!" .format(key, type_)) continue # load public id in the registry if any name = key configuration_file_path = get_configuration_file_path(type_, name) current_public_id = get_public_id_from_yaml( configuration_file_path) deployed_public_id = public_id_in_registry(type_, name) difference = minor_version_difference(current_public_id, deployed_public_id) # check if the public ids of the local package and the package in the registry are already the same. package_info = f"Package `{name}` of type `{type_}`" public_id_info = f"current id `{current_public_id}` and deployed id `{deployed_public_id}`" if difference == 0: print(f"{package_info} needs to be bumped!") result.add(PackageId(type_[:-1], current_public_id)) elif difference == 1: print(f"{package_info} already at correct version!") continue else: print(f"{package_info} has {public_id_info}. Error!") sys.exit(1) return result
def find_all_packages_ids() -> Set[PackageId]: """Find all packages ids.""" package_ids: Set[PackageId] = set() packages_dir = Path("packages") for configuration_file in packages_dir.glob("*/*/*/*.yaml"): package_type = PackageType(configuration_file.parts[2][:-1]) package_public_id = get_public_id_from_yaml(configuration_file) package_id = PackageId(package_type, package_public_id) package_ids.add(package_id) return package_ids
def _get_item_requirements( item: PackageConfiguration, ) -> Generator[PackageId, None, None]: """ List all the requiemenents for item provided. :return: generator with package ids: (type, public_id) """ for item_type in map(str, ComponentType): items = getattr(item, f"{item_type}s", set()) for item_public_id in items: yield PackageId(item_type, item_public_id)
def setup(cls): """Set the test up.""" super(TestRemoveAndDependencies, cls).setup() cls.DEPENDENCY_PACKAGE_ID = PackageId( cls.DEPENDENCY_TYPE, cls.DEPENDENCY_PUBLIC_ID ) result = cls.runner.invoke( cli, ["-v", "DEBUG", "add", "--local", cls.ITEM_TYPE, str(cls.ITEM_PUBLIC_ID)], standalone_mode=False, ) assert result.exit_code == 0
def get_all_package_ids() -> Set[PackageId]: """Get all the package ids in the local repository.""" result: Set[PackageId] = set() now = get_hashes_from_current_release() now_by_type = split_hashes_by_type(now) for type_, name_to_hashes in now_by_type.items(): for name, _ in name_to_hashes.items(): if name in TEST_PROTOCOLS: continue configuration_file_path = get_configuration_file_path(type_, name) public_id = get_public_id_from_yaml(configuration_file_path) package_id = PackageId(PackageType(type_[:-1]), public_id) result.add(package_id) return result
def find_all_packages_ids() -> Set[PackageId]: """Find all packages ids.""" package_ids: Set[PackageId] = set() packages_dir = Path("packages") config_files = [ path for path in packages_dir.glob("*/*/*/*.yaml") if any([file in str(path) for file in CONFIG_FILE_NAMES]) ] for configuration_file in chain(config_files, default_config_file_paths()): package_type = PackageType(configuration_file.parts[-3][:-1]) package_public_id = get_public_id_from_yaml(configuration_file) package_id = PackageId(package_type, package_public_id) package_ids.add(package_id) return package_ids
def _get_item_requirements(self, item: PackageConfiguration, ignore_non_vendor: bool = False ) -> Generator[PackageId, None, None]: """ List all the requirements for item provided. :return: generator with package ids: (type, public_id) """ for item_type in map(str, ComponentType): items = getattr(item, f"{item_type}s", set()) for item_public_id in items: if ignore_non_vendor and is_item_present( self._ctx.cwd, self._ctx.agent_config, item_type, item_public_id, is_vendor=False, ): continue yield PackageId(item_type, item_public_id)
def test_agent_config_to_json_with_optional_configurations(): """Test agent config to json with optional configurations.""" agent_config = AgentConfig( "name", "author", period=0.1, execution_timeout=1.0, max_reactions=100, decision_maker_handler=dict(dotted_path="", file_path=""), error_handler=dict(dotted_path="", file_path=""), skill_exception_policy="propagate", connection_exception_policy="propagate", default_routing={"author/name:0.1.0": "author/name:0.1.0"}, currency_denominations={"fetchai": "fet"}, loop_mode="sync", runtime_mode="async", storage_uri="some_uri_to_storage", ) agent_config.default_connection = "author/name:0.1.0" agent_config.default_ledger = DEFAULT_LEDGER agent_config.json assert agent_config.package_id == PackageId.from_uri_path( "agent/author/name/0.1.0")
def setup(self): """Set the test up.""" self.runner = CliRunner() self.agent_name = "myagent" self.cwd = os.getcwd() self.t = tempfile.mkdtemp() # copy the 'packages' directory in the parent of the agent folder. shutil.copytree(Path(CUR_PATH, "..", "packages"), Path(self.t, "packages")) os.chdir(self.t) result = self.runner.invoke( cli, [*CLI_LOG_OPTION, "init", "--local", "--author", AUTHOR], standalone_mode=False, ) assert result.exit_code == 0 result = self.runner.invoke( cli, [*CLI_LOG_OPTION, "create", "--local", self.agent_name], standalone_mode=False, ) assert result.exit_code == 0 os.chdir(self.agent_name) # add connection first time self.DEPENDENCY_PACKAGE_ID = PackageId(self.DEPENDENCY_TYPE, self.DEPENDENCY_PUBLIC_ID) result = self.runner.invoke( cli, [ "-v", "DEBUG", "add", "--local", self.ITEM_TYPE, str(self.ITEM_PUBLIC_ID) ], standalone_mode=False, catch_exceptions=False, )
def _get_item_folder(self) -> Path: """Get item package folder.""" return Path(self.cwd) / ItemRemoveHelper.get_component_directory( PackageId(self.item_type, self.item_id) )
def _add_package_type(package_type, public_id_str): return PackageId(package_type, PublicId.from_str(public_id_str))
def test_package_id_to_uri_path(): """Test PackageId.to_uri_path""" package_id = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.1.0")) assert package_id.to_uri_path == "protocol/author/name/0.1.0"
def test_package_id_str(): """Test PackageId.__str__""" package_id = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.1.0")) assert str(package_id) == "(protocol, author/name:0.1.0)"
def test_package_id_version(): """Test PackageId.version""" package_id = PackageId(PackageType.PROTOCOL, PublicId("author", "name", "0.1.0")) assert package_id.version == "0.1.0"
def _eject_item( ctx: Context, item_type: str, public_id: PublicId, quiet: bool = True, with_symlinks: bool = False, ) -> None: """ Eject item from installed (vendor) to custom folder. :param ctx: context object. :param item_type: item type. :param public_id: item public ID. :param quiet: if false, the function will ask the user in case of recursive eject. :return: None :raises: ClickException if item is absent at source path or present at destenation path. """ # we know cli_author is set because of the above checks. cli_author: str = cast(str, ctx.config.get("cli_author")) item_type_plural = item_type + "s" if not is_item_present( ctx.cwd, ctx.agent_config, item_type, public_id, is_vendor=True, with_version=True, ): # pragma: no cover raise click.ClickException( f"{item_type.title()} {public_id} not found in agent's vendor items." ) src = get_package_path(ctx.cwd, item_type, public_id) dst = get_package_path(ctx.cwd, item_type, public_id, is_vendor=False) if is_item_present(ctx.cwd, ctx.agent_config, item_type, public_id, is_vendor=False): # pragma: no cover raise click.ClickException( f"{item_type.title()} {public_id} is already a non-vendor package." ) configuration = load_item_config(item_type, Path(src)) if public_id.package_version.is_latest: # get 'concrete' public id, in case it is 'latest' component_prefix = ComponentType( item_type), public_id.author, public_id.name component_id = get_latest_component_id_from_prefix( ctx.agent_config, component_prefix) # component id is necessarily found, due to the checks above. public_id = cast(ComponentId, component_id).public_id package_id = PackageId(PackageType(item_type), public_id) click.echo( f"Ejecting item {package_id.package_type.value} {str(package_id.public_id)}" ) # first, eject all the vendor packages that depend on this item_remover = ItemRemoveHelper(ctx, ignore_non_vendor=True) reverse_dependencies = ( item_remover.get_agent_dependencies_with_reverse_dependencies()) reverse_reachable_dependencies = reachable_nodes(reverse_dependencies, {package_id}) # the reversed topological order of a graph # is the topological order of the reverse graph. eject_order = list( reversed(find_topological_order(reverse_reachable_dependencies))) eject_order.remove(package_id) if len(eject_order) > 0 and not quiet: click.echo( f"The following vendor packages will be ejected: {eject_order}") answer = click.confirm("Do you want to proceed?") if not answer: click.echo("Aborted.") return for dependency_package_id in eject_order: # 'dependency_package_id' depends on 'package_id', # so we need to eject it first _eject_item( ctx, dependency_package_id.package_type.value, dependency_package_id.public_id, quiet=True, ) # copy the vendor package into the non-vendor packages ctx.clean_paths.append(dst) copy_package_directory(Path(src), dst) new_public_id = PublicId(cli_author, public_id.name, DEFAULT_VERSION) current_version = Version(aea.__version__) new_aea_range = ( configuration.aea_version if configuration.aea_version_specifiers.contains(current_version) else compute_specifier_from_version(current_version)) update_item_config( item_type, Path(dst), author=new_public_id.author, version=new_public_id.version, aea_version=new_aea_range, ) update_item_public_id_in_init(item_type, Path(dst), new_public_id) shutil.rmtree(src) # update references in all the other packages component_type = ComponentType(item_type_plural[:-1]) old_component_id = ComponentId(component_type, public_id) new_component_id = ComponentId(component_type, new_public_id) update_references(ctx, {old_component_id: new_component_id}) # need to reload agent configuration with the updated references try_to_load_agent_config(ctx) # replace import statements in all the non-vendor packages replace_all_import_statements(Path(ctx.cwd), ComponentType(item_type), public_id, new_public_id) # fingerprint all (non-vendor) packages fingerprint_all(ctx) if with_symlinks: click.echo( "Adding symlinks from vendor to non-vendor and packages to vendor folders." ) create_symlink_vendor_to_local(ctx, item_type, new_public_id) create_symlink_packages_to_vendor(ctx) click.echo( f"Successfully ejected {item_type} {public_id} to {dst} as {new_public_id}." )
def _add_package_type(package_type: PackageType, public_id_str: str) -> PackageId: return PackageId(package_type, PublicId.from_str(public_id_str))
def extract_package_id(match: Match) -> PackageId: package_type, package = match.group(1), match.group(2) package_id = PackageId(PackageType(package_type), PublicId.from_str(package)) return package_id
def extract_package_id(match: Match) -> PackageId: package_public_id = match.group(1) package_id = PackageId(PackageType.AGENT, PublicId.from_str(package_public_id)) return package_id
def is_non_vendor(self) -> bool: """Check is package specified is non vendor.""" path = ItemRemoveHelper.get_component_directory( PackageId(self.item_type, self.item_public_id) ) return "vendor" not in Path(path).parts[:2]