def _get_item_details(ctx, item_type) -> List[Dict]: """Return a list of item details, given the item type.""" result = [] item_type_plural = item_type + "s" public_ids = getattr(ctx.agent_config, item_type_plural) # type: Set[PublicId] default_file_name = _get_default_configuration_file_name_from_type( item_type) for public_id in public_ids: # first, try to retrieve the item from the vendor directory. configuration_filepath = Path( ctx.cwd, "vendor", public_id.author, item_type_plural, public_id.name, default_file_name, ) # otherwise, if it does not exist, retrieve the item from the agent custom packages if not configuration_filepath.exists(): configuration_filepath = Path(ctx.cwd, item_type_plural, public_id.name, default_file_name) configuration_loader = ConfigLoader.from_configuration_type( PackageType(item_type)) details = retrieve_details(public_id.name, configuration_loader, str(configuration_filepath)) result.append(details) return result
def ipfs_hashing( client: ipfshttpclient.Client, configuration: PackageConfiguration, package_type: PackageType, ) -> Tuple[str, str, List[Dict]]: """ Hashes a package and its components. :param client: a connected IPFS client. :param configuration: the package configuration. :param package_type: the package type. :return: the identifier of the hash (e.g. 'fetchai/protocols/default') | and the hash of the whole package. """ # hash again to get outer hash (this time all files) # we still need to ignore some files # use ignore patterns somehow # ignore_patterns = configuration.fingerprint_ignore_patterns # noqa: E800 assert configuration.directory is not None result_list = client.add( configuration.directory, recursive=True, period_special=False, follow_symlinks=False, ) key = os.path.join( configuration.author, package_type.to_plural(), configuration.directory.name, ) # check that the last result of the list is for the whole package directory assert result_list[-1]["Name"] == configuration.directory.name directory_hash = result_list[-1]["Hash"] return key, directory_hash, result_list
def update_references(ctx: Context, replacements: Dict[ComponentId, ComponentId]): """ Update references across an AEA project. Caveat: the update is done in a sequential manner. There is no check of multiple updates, due to the occurrence of transitive relations. E.g. replacements as {c1: c2, c2: c3} might lead to c1 replaced with c3 instead of c2. :param ctx: the context. :param replacements: mapping from old component ids to new component ids. :return: None. """ # preprocess replacement so to index them by component type replacements_by_type: Dict[ComponentType, Dict[PublicId, PublicId]] = {} for old, new in replacements.items(): replacements_by_type.setdefault(old.component_type, {})[old.public_id] = new.public_id aea_project_root = Path(ctx.cwd) # update agent configuration agent_config = load_item_config(PackageType.AGENT.value, aea_project_root) replace_component_ids(agent_config, replacements_by_type) dump_item_config(agent_config, aea_project_root) # update every (non-vendor) AEA package. for package_path in get_non_vendor_package_path(aea_project_root): package_type = PackageType(package_path.parent.name[:-1]) package_config = load_item_config(package_type.value, package_path) replace_component_ids(package_config, replacements_by_type) dump_item_config(package_config, package_path)
def from_package_type( cls, configuration_type: Union[PackageType, str]) -> "ConfigLoader": """ Get a config loader from the configuration type. :param configuration_type: the configuration type """ configuration_type = PackageType(configuration_type) return cls._from_configuration_type_to_loaders[configuration_type]
def find_all_packages_ids() -> Set[PackageId]: """Find all packages ids.""" package_ids: Set[PackageId] = set() for configuration_file in find_all_configuration_files(): package_type = PackageType(configuration_file.parts[-3][:-1]) package_public_id = get_public_id_from_yaml(configuration_file) package_id = PackageId(package_type, package_public_id) package_ids.add(package_id) return package_ids
def find_all_packages_ids() -> Set[PackageId]: """Find all packages ids.""" package_ids: Set[PackageId] = set() packages_dir = Path("packages") for configuration_file in packages_dir.glob("*/*/*/*.yaml"): package_type = PackageType(configuration_file.parts[2][:-1]) package_public_id = get_public_id_from_yaml(configuration_file) package_id = PackageId(package_type, package_public_id) package_ids.add(package_id) return package_ids
def from_package_type( cls, configuration_type: Union[PackageType, str]) -> "ConfigLoader": """ Get a config loader from the configuration type. :param configuration_type: the configuration type """ config_class: Type[ PackageConfiguration] = PACKAGE_TYPE_TO_CONFIG_CLASS[PackageType( configuration_type)] return ConfigLoader(config_class.schema, config_class)
def update_aea_version_in_nonvendor_packages(cwd: str) -> None: """ Update aea_version in non-vendor packages. :param cwd: the current working directory. :return: None """ for package_path in get_non_vendor_package_path(Path(cwd)): package_type = PackageType(package_path.parent.name[:-1]) package_config = load_item_config(package_type.value, package_path) update_aea_version_range(package_config) dump_item_config(package_config, package_path)
def find_item_locally( ctx: Context, item_type: str, item_public_id: PublicId) -> Tuple[Path, ComponentConfiguration]: """ Find an item in the local registry. :param ctx: the CLI context. :param item_type: the type of the item to load. One of: protocols, connections, skills :param item_public_id: the public id of the item to find. :return: tuple of path to the package directory (either in registry or in aea directory) and component configuration :raises SystemExit: if the search fails. """ item_type_plural = item_type + "s" item_name = item_public_id.name # check in registry registry_path = (os.path.join(ctx.cwd, ctx.agent_config.registry_path) if ctx.registry_path is None else ctx.registry_path) package_path = Path(registry_path, item_public_id.author, item_type_plural, item_name) config_file_name = _get_default_configuration_file_name_from_type( item_type) item_configuration_filepath = package_path / config_file_name if not item_configuration_filepath.exists(): raise click.ClickException("Cannot find {}: '{}'.".format( item_type, item_public_id)) # try to load the item configuration file try: item_configuration_loader = ConfigLoader.from_configuration_type( PackageType(item_type)) with item_configuration_filepath.open() as fp: item_configuration = item_configuration_loader.load(fp) except ValidationError as e: raise click.ClickException( "{} configuration file not valid: {}".format( item_type.capitalize(), str(e))) # check that the configuration file of the found package matches the expected author and version. version = item_configuration.version author = item_configuration.author if item_public_id.author != author or ( not item_public_id.package_version.is_latest and item_public_id.version != version): raise click.ClickException( "Cannot find {} with author and version specified.".format( item_type)) return package_path, item_configuration
def get_all_package_ids() -> Set[PackageId]: """Get all the package ids in the local repository.""" result: Set[PackageId] = set() now = get_hashes_from_current_release() now_by_type = split_hashes_by_type(now) for type_, name_to_hashes in now_by_type.items(): for name, _ in name_to_hashes.items(): if name in TEST_PROTOCOLS: continue configuration_file_path = get_configuration_file_path(type_, name) public_id = get_public_id_from_yaml(configuration_file_path) package_id = PackageId(PackageType(type_[:-1]), public_id) result.add(package_id) return result
def find_all_packages_ids() -> Set[PackageId]: """Find all packages ids.""" package_ids: Set[PackageId] = set() packages_dir = Path("packages") config_files = [ path for path in packages_dir.glob("*/*/*/*.yaml") if any([file in str(path) for file in CONFIG_FILE_NAMES]) ] for configuration_file in chain(config_files, default_config_file_paths()): package_type = PackageType(configuration_file.parts[-3][:-1]) package_public_id = get_public_id_from_yaml(configuration_file) package_id = PackageId(package_type, package_public_id) package_ids.add(package_id) return package_ids
def load_item_config(item_type: str, package_path: Path) -> PackageConfiguration: """ Load item configuration. :param item_type: type of item. :param package_path: path to package from which config should be loaded. :return: configuration object. """ configuration_file_name = _get_default_configuration_file_name_from_type(item_type) configuration_path = package_path / configuration_file_name configuration_loader = ConfigLoader.from_configuration_type(PackageType(item_type)) with open_file(configuration_path) as file_input: item_config = configuration_loader.load(file_input) return item_config
def find_item_in_distribution( # pylint: disable=unused-argument ctx: Context, item_type: str, item_public_id: PublicId) -> Path: """ Find an item in the AEA directory. :param ctx: the CLI context. :param item_type: the type of the item to load. One of: protocols, connections, skills :param item_public_id: the public id of the item to find. :return: path to the package directory (either in registry or in aea directory). :raises SystemExit: if the search fails. """ item_type_plural = item_type + "s" item_name = item_public_id.name # check in aea dir registry_path = AEA_DIR package_path = Path(registry_path, item_type_plural, item_name) config_file_name = _get_default_configuration_file_name_from_type( item_type) item_configuration_filepath = package_path / config_file_name if not item_configuration_filepath.exists(): raise click.ClickException("Cannot find {}: '{}'.".format( item_type, item_public_id)) # try to load the item configuration file try: item_configuration_loader = ConfigLoader.from_configuration_type( PackageType(item_type)) with item_configuration_filepath.open() as fp: item_configuration = item_configuration_loader.load(fp) except ValidationError as e: raise click.ClickException( "{} configuration file not valid: {}".format( item_type.capitalize(), str(e))) # check that the configuration file of the found package matches the expected author and version. version = item_configuration.version author = item_configuration.author if item_public_id.author != author or ( not item_public_id.package_version.is_latest and item_public_id.version != version): raise click.ClickException( "Cannot find {} with author and version specified.".format( item_type)) return package_path # pragma: no cover
def _update_non_vendor_package( package_path: Path, replacements: Dict[ComponentType, Dict[Tuple[str, str], PublicId]], ) -> None: """Update a single non-vendor package.""" """ A path to a non-vendor package in an AEA project is of the form: .../aea-project-path/package-type/package-name/ so we need to get the second-to-last part of the path to infer the type. """ type_plural = package_path.parts[-2] loader = ConfigLoaders.from_package_type(PackageType(type_plural[:-1])) path_to_config = ( package_path / loader.configuration_class.default_configuration_filename ) with path_to_config.open() as file_in: component_config = loader.load(file_in) update_dependencies(component_config, replacements) with path_to_config.open(mode="w") as file_out: loader.dump(component_config, file_out)
def package_type_and_path(package_path: Path) -> Tuple[PackageType, Path]: """Extract the package type from the path.""" item_type_plural = package_path.parent.name item_type_singular = item_type_plural[:-1] return PackageType(item_type_singular), package_path
def _eject_item( ctx: Context, item_type: str, public_id: PublicId, quiet: bool = True, with_symlinks: bool = False, ) -> None: """ Eject item from installed (vendor) to custom folder. :param ctx: context object. :param item_type: item type. :param public_id: item public ID. :param quiet: if false, the function will ask the user in case of recursive eject. :return: None :raises: ClickException if item is absent at source path or present at destenation path. """ # we know cli_author is set because of the above checks. cli_author: str = cast(str, ctx.config.get("cli_author")) item_type_plural = item_type + "s" if not is_item_present( ctx.cwd, ctx.agent_config, item_type, public_id, is_vendor=True, with_version=True, ): # pragma: no cover raise click.ClickException( f"{item_type.title()} {public_id} not found in agent's vendor items." ) src = get_package_path(ctx.cwd, item_type, public_id) dst = get_package_path(ctx.cwd, item_type, public_id, is_vendor=False) if is_item_present(ctx.cwd, ctx.agent_config, item_type, public_id, is_vendor=False): # pragma: no cover raise click.ClickException( f"{item_type.title()} {public_id} is already a non-vendor package." ) configuration = load_item_config(item_type, Path(src)) if public_id.package_version.is_latest: # get 'concrete' public id, in case it is 'latest' component_prefix = ComponentType( item_type), public_id.author, public_id.name component_id = get_latest_component_id_from_prefix( ctx.agent_config, component_prefix) # component id is necessarily found, due to the checks above. public_id = cast(ComponentId, component_id).public_id package_id = PackageId(PackageType(item_type), public_id) click.echo( f"Ejecting item {package_id.package_type.value} {str(package_id.public_id)}" ) # first, eject all the vendor packages that depend on this item_remover = ItemRemoveHelper(ctx, ignore_non_vendor=True) reverse_dependencies = ( item_remover.get_agent_dependencies_with_reverse_dependencies()) reverse_reachable_dependencies = reachable_nodes(reverse_dependencies, {package_id}) # the reversed topological order of a graph # is the topological order of the reverse graph. eject_order = list( reversed(find_topological_order(reverse_reachable_dependencies))) eject_order.remove(package_id) if len(eject_order) > 0 and not quiet: click.echo( f"The following vendor packages will be ejected: {eject_order}") answer = click.confirm("Do you want to proceed?") if not answer: click.echo("Aborted.") return for dependency_package_id in eject_order: # 'dependency_package_id' depends on 'package_id', # so we need to eject it first _eject_item( ctx, dependency_package_id.package_type.value, dependency_package_id.public_id, quiet=True, ) # copy the vendor package into the non-vendor packages ctx.clean_paths.append(dst) copy_package_directory(Path(src), dst) new_public_id = PublicId(cli_author, public_id.name, DEFAULT_VERSION) current_version = Version(aea.__version__) new_aea_range = ( configuration.aea_version if configuration.aea_version_specifiers.contains(current_version) else compute_specifier_from_version(current_version)) update_item_config( item_type, Path(dst), author=new_public_id.author, version=new_public_id.version, aea_version=new_aea_range, ) update_item_public_id_in_init(item_type, Path(dst), new_public_id) shutil.rmtree(src) # update references in all the other packages component_type = ComponentType(item_type_plural[:-1]) old_component_id = ComponentId(component_type, public_id) new_component_id = ComponentId(component_type, new_public_id) update_references(ctx, {old_component_id: new_component_id}) # need to reload agent configuration with the updated references try_to_load_agent_config(ctx) # replace import statements in all the non-vendor packages replace_all_import_statements(Path(ctx.cwd), ComponentType(item_type), public_id, new_public_id) # fingerprint all (non-vendor) packages fingerprint_all(ctx) if with_symlinks: click.echo( "Adding symlinks from vendor to non-vendor and packages to vendor folders." ) create_symlink_vendor_to_local(ctx, item_type, new_public_id) create_symlink_packages_to_vendor(ctx) click.echo( f"Successfully ejected {item_type} {public_id} to {dst} as {new_public_id}." )
def from_configuration_type( cls, configuration_type: Union[PackageType, str] ) -> "ConfigLoader": """Get the configuration loader from the type.""" configuration_type = PackageType(configuration_type) return ConfigLoaders.from_package_type(configuration_type)
def from_package_type( cls, configuration_type: Union[PackageType, str] ) -> "ConfigLoader": configuration_type = PackageType(configuration_type) return cls._from_configuration_type_to_loaders[configuration_type]
def extract_package_id(match: Match) -> PackageId: package_type, package = match.group(1), match.group(2) package_id = PackageId(PackageType(package_type), PublicId.from_str(package)) return package_id
def _add_item(click_context, item_type, item_public_id) -> None: """ Add an item. :param click_context: the click context. :param item_type: the item type. :param item_public_id: the item public id. :return: None """ ctx = cast(Context, click_context.obj) agent_name = cast(str, ctx.agent_config.agent_name) item_type_plural = item_type + "s" supported_items = getattr(ctx.agent_config, item_type_plural) is_local = ctx.config.get("is_local") click.echo( "Adding {} '{}' to the agent '{}'...".format( item_type, item_public_id, agent_name ) ) # check if we already have an item with the same name logger.debug( "{} already supported by the agent: {}".format( item_type_plural.capitalize(), supported_items ) ) if _is_item_present(item_type, item_public_id, ctx): raise click.ClickException( "A {} with id '{}/{}' already exists. Aborting...".format( item_type, item_public_id.author, item_public_id.name ) ) # find and add protocol dest_path = get_package_dest_path( ctx, item_public_id.author, item_type_plural, item_public_id.name ) ctx.clean_paths.append(dest_path) if item_public_id in [DEFAULT_CONNECTION, DEFAULT_PROTOCOL, DEFAULT_SKILL]: source_path = find_item_in_distribution(ctx, item_type, item_public_id) package_path = copy_package_directory( ctx, source_path, item_type, item_public_id.name, item_public_id.author, dest_path, ) elif is_local: source_path = find_item_locally(ctx, item_type, item_public_id) package_path = copy_package_directory( ctx, source_path, item_type, item_public_id.name, item_public_id.author, dest_path, ) else: package_path = fetch_package( item_type, public_id=item_public_id, cwd=ctx.cwd, dest=dest_path ) configuration_file_name = _get_default_configuration_file_name_from_type(item_type) configuration_path = package_path / configuration_file_name configuration_loader = ConfigLoader.from_configuration_type(PackageType(item_type)) item_configuration = configuration_loader.load(configuration_path.open()) _validate_fingerprint(package_path, item_configuration) if item_type in {"connection", "skill"}: _add_protocols(click_context, item_configuration.protocols) if item_type == "skill": for contract_public_id in item_configuration.contracts: if contract_public_id not in ctx.agent_config.contracts: _add_item(click_context, "contract", contract_public_id) # add the item to the configurations. logger.debug( "Registering the {} into {}".format(item_type, DEFAULT_AEA_CONFIG_FILE) ) supported_items.add(item_public_id) ctx.agent_loader.dump( ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w") )