def _get_item_details(ctx, item_type) -> List[Dict]:
"""Return a list of item details, given the item type."""
result = []
item_type_plural = item_type + "s"
public_ids = getattr(ctx.agent_config,
item_type_plural) # type: Set[PublicId]
default_file_name = _get_default_configuration_file_name_from_type(
item_type)
for public_id in public_ids:
# first, try to retrieve the item from the vendor directory.
configuration_filepath = Path(
ctx.cwd,
"vendor",
public_id.author,
item_type_plural,
public_id.name,
default_file_name,
)
# otherwise, if it does not exist, retrieve the item from the agent custom packages
if not configuration_filepath.exists():
configuration_filepath = Path(ctx.cwd, item_type_plural,
public_id.name, default_file_name)
configuration_loader = ConfigLoader.from_configuration_type(
PackageType(item_type))
details = retrieve_details(public_id.name, configuration_loader,
str(configuration_filepath))
result.append(details)
return result
def ipfs_hashing(
client: ipfshttpclient.Client,
configuration: PackageConfiguration,
package_type: PackageType,
) -> Tuple[str, str, List[Dict]]:
"""
Hashes a package and its components.
:param client: a connected IPFS client.
:param configuration: the package configuration.
:param package_type: the package type.
:return: the identifier of the hash (e.g. 'fetchai/protocols/default')
| and the hash of the whole package.
"""
# hash again to get outer hash (this time all files)
# we still need to ignore some files
# use ignore patterns somehow
# ignore_patterns = configuration.fingerprint_ignore_patterns # noqa: E800
assert configuration.directory is not None
result_list = client.add(
configuration.directory,
recursive=True,
period_special=False,
follow_symlinks=False,
)
key = os.path.join(
configuration.author,
package_type.to_plural(),
configuration.directory.name,
)
# check that the last result of the list is for the whole package directory
assert result_list[-1]["Name"] == configuration.directory.name
directory_hash = result_list[-1]["Hash"]
return key, directory_hash, result_list
def update_references(ctx: Context, replacements: Dict[ComponentId,
ComponentId]):
"""
Update references across an AEA project.
Caveat: the update is done in a sequential manner. There is no check
of multiple updates, due to the occurrence of transitive relations.
E.g. replacements as {c1: c2, c2: c3} might lead to c1 replaced with c3
instead of c2.
:param ctx: the context.
:param replacements: mapping from old component ids to new component ids.
:return: None.
"""
# preprocess replacement so to index them by component type
replacements_by_type: Dict[ComponentType, Dict[PublicId, PublicId]] = {}
for old, new in replacements.items():
replacements_by_type.setdefault(old.component_type,
{})[old.public_id] = new.public_id
aea_project_root = Path(ctx.cwd)
# update agent configuration
agent_config = load_item_config(PackageType.AGENT.value, aea_project_root)
replace_component_ids(agent_config, replacements_by_type)
dump_item_config(agent_config, aea_project_root)
# update every (non-vendor) AEA package.
for package_path in get_non_vendor_package_path(aea_project_root):
package_type = PackageType(package_path.parent.name[:-1])
package_config = load_item_config(package_type.value, package_path)
replace_component_ids(package_config, replacements_by_type)
dump_item_config(package_config, package_path)
def from_package_type(
cls, configuration_type: Union[PackageType,
str]) -> "ConfigLoader":
"""
Get a config loader from the configuration type.
:param configuration_type: the configuration type
"""
configuration_type = PackageType(configuration_type)
return cls._from_configuration_type_to_loaders[configuration_type]
def find_all_packages_ids() -> Set[PackageId]:
"""Find all packages ids."""
package_ids: Set[PackageId] = set()
for configuration_file in find_all_configuration_files():
package_type = PackageType(configuration_file.parts[-3][:-1])
package_public_id = get_public_id_from_yaml(configuration_file)
package_id = PackageId(package_type, package_public_id)
package_ids.add(package_id)
return package_ids
def find_all_packages_ids() -> Set[PackageId]:
"""Find all packages ids."""
package_ids: Set[PackageId] = set()
packages_dir = Path("packages")
for configuration_file in packages_dir.glob("*/*/*/*.yaml"):
package_type = PackageType(configuration_file.parts[2][:-1])
package_public_id = get_public_id_from_yaml(configuration_file)
package_id = PackageId(package_type, package_public_id)
package_ids.add(package_id)
return package_ids
def from_package_type(
cls, configuration_type: Union[PackageType,
str]) -> "ConfigLoader":
"""
Get a config loader from the configuration type.
:param configuration_type: the configuration type
"""
config_class: Type[
PackageConfiguration] = PACKAGE_TYPE_TO_CONFIG_CLASS[PackageType(
configuration_type)]
return ConfigLoader(config_class.schema, config_class)
def update_aea_version_in_nonvendor_packages(cwd: str) -> None:
"""
Update aea_version in non-vendor packages.
:param cwd: the current working directory.
:return: None
"""
for package_path in get_non_vendor_package_path(Path(cwd)):
package_type = PackageType(package_path.parent.name[:-1])
package_config = load_item_config(package_type.value, package_path)
update_aea_version_range(package_config)
dump_item_config(package_config, package_path)
def find_item_locally(
ctx: Context, item_type: str,
item_public_id: PublicId) -> Tuple[Path, ComponentConfiguration]:
"""
Find an item in the local registry.
:param ctx: the CLI context.
:param item_type: the type of the item to load. One of: protocols, connections, skills
:param item_public_id: the public id of the item to find.
:return: tuple of path to the package directory (either in registry or in aea directory) and component configuration
:raises SystemExit: if the search fails.
"""
item_type_plural = item_type + "s"
item_name = item_public_id.name
# check in registry
registry_path = (os.path.join(ctx.cwd, ctx.agent_config.registry_path)
if ctx.registry_path is None else ctx.registry_path)
package_path = Path(registry_path, item_public_id.author, item_type_plural,
item_name)
config_file_name = _get_default_configuration_file_name_from_type(
item_type)
item_configuration_filepath = package_path / config_file_name
if not item_configuration_filepath.exists():
raise click.ClickException("Cannot find {}: '{}'.".format(
item_type, item_public_id))
# try to load the item configuration file
try:
item_configuration_loader = ConfigLoader.from_configuration_type(
PackageType(item_type))
with item_configuration_filepath.open() as fp:
item_configuration = item_configuration_loader.load(fp)
except ValidationError as e:
raise click.ClickException(
"{} configuration file not valid: {}".format(
item_type.capitalize(), str(e)))
# check that the configuration file of the found package matches the expected author and version.
version = item_configuration.version
author = item_configuration.author
if item_public_id.author != author or (
not item_public_id.package_version.is_latest
and item_public_id.version != version):
raise click.ClickException(
"Cannot find {} with author and version specified.".format(
item_type))
return package_path, item_configuration
def get_all_package_ids() -> Set[PackageId]:
"""Get all the package ids in the local repository."""
result: Set[PackageId] = set()
now = get_hashes_from_current_release()
now_by_type = split_hashes_by_type(now)
for type_, name_to_hashes in now_by_type.items():
for name, _ in name_to_hashes.items():
if name in TEST_PROTOCOLS:
continue
configuration_file_path = get_configuration_file_path(type_, name)
public_id = get_public_id_from_yaml(configuration_file_path)
package_id = PackageId(PackageType(type_[:-1]), public_id)
result.add(package_id)
return result
def find_all_packages_ids() -> Set[PackageId]:
"""Find all packages ids."""
package_ids: Set[PackageId] = set()
packages_dir = Path("packages")
config_files = [
path for path in packages_dir.glob("*/*/*/*.yaml")
if any([file in str(path) for file in CONFIG_FILE_NAMES])
]
for configuration_file in chain(config_files, default_config_file_paths()):
package_type = PackageType(configuration_file.parts[-3][:-1])
package_public_id = get_public_id_from_yaml(configuration_file)
package_id = PackageId(package_type, package_public_id)
package_ids.add(package_id)
return package_ids
def load_item_config(item_type: str, package_path: Path) -> PackageConfiguration:
"""
Load item configuration.
:param item_type: type of item.
:param package_path: path to package from which config should be loaded.
:return: configuration object.
"""
configuration_file_name = _get_default_configuration_file_name_from_type(item_type)
configuration_path = package_path / configuration_file_name
configuration_loader = ConfigLoader.from_configuration_type(PackageType(item_type))
with open_file(configuration_path) as file_input:
item_config = configuration_loader.load(file_input)
return item_config
def find_item_in_distribution( # pylint: disable=unused-argument
ctx: Context, item_type: str, item_public_id: PublicId) -> Path:
"""
Find an item in the AEA directory.
:param ctx: the CLI context.
:param item_type: the type of the item to load. One of: protocols, connections, skills
:param item_public_id: the public id of the item to find.
:return: path to the package directory (either in registry or in aea directory).
:raises SystemExit: if the search fails.
"""
item_type_plural = item_type + "s"
item_name = item_public_id.name
# check in aea dir
registry_path = AEA_DIR
package_path = Path(registry_path, item_type_plural, item_name)
config_file_name = _get_default_configuration_file_name_from_type(
item_type)
item_configuration_filepath = package_path / config_file_name
if not item_configuration_filepath.exists():
raise click.ClickException("Cannot find {}: '{}'.".format(
item_type, item_public_id))
# try to load the item configuration file
try:
item_configuration_loader = ConfigLoader.from_configuration_type(
PackageType(item_type))
with item_configuration_filepath.open() as fp:
item_configuration = item_configuration_loader.load(fp)
except ValidationError as e:
raise click.ClickException(
"{} configuration file not valid: {}".format(
item_type.capitalize(), str(e)))
# check that the configuration file of the found package matches the expected author and version.
version = item_configuration.version
author = item_configuration.author
if item_public_id.author != author or (
not item_public_id.package_version.is_latest
and item_public_id.version != version):
raise click.ClickException(
"Cannot find {} with author and version specified.".format(
item_type))
return package_path # pragma: no cover
def _update_non_vendor_package(
package_path: Path,
replacements: Dict[ComponentType, Dict[Tuple[str, str], PublicId]],
) -> None:
"""Update a single non-vendor package."""
"""
A path to a non-vendor package in an AEA project is of the form:
.../aea-project-path/package-type/package-name/
so we need to get the second-to-last part of the path to infer the type.
"""
type_plural = package_path.parts[-2]
loader = ConfigLoaders.from_package_type(PackageType(type_plural[:-1]))
path_to_config = (
package_path / loader.configuration_class.default_configuration_filename
)
with path_to_config.open() as file_in:
component_config = loader.load(file_in)
update_dependencies(component_config, replacements)
with path_to_config.open(mode="w") as file_out:
loader.dump(component_config, file_out)
def package_type_and_path(package_path: Path) -> Tuple[PackageType, Path]:
"""Extract the package type from the path."""
item_type_plural = package_path.parent.name
item_type_singular = item_type_plural[:-1]
return PackageType(item_type_singular), package_path
def _eject_item(
ctx: Context,
item_type: str,
public_id: PublicId,
quiet: bool = True,
with_symlinks: bool = False,
) -> None:
"""
Eject item from installed (vendor) to custom folder.
:param ctx: context object.
:param item_type: item type.
:param public_id: item public ID.
:param quiet: if false, the function will ask the user in case of recursive eject.
:return: None
:raises: ClickException if item is absent at source path or present at destenation path.
"""
# we know cli_author is set because of the above checks.
cli_author: str = cast(str, ctx.config.get("cli_author"))
item_type_plural = item_type + "s"
if not is_item_present(
ctx.cwd,
ctx.agent_config,
item_type,
public_id,
is_vendor=True,
with_version=True,
): # pragma: no cover
raise click.ClickException(
f"{item_type.title()} {public_id} not found in agent's vendor items."
)
src = get_package_path(ctx.cwd, item_type, public_id)
dst = get_package_path(ctx.cwd, item_type, public_id, is_vendor=False)
if is_item_present(ctx.cwd,
ctx.agent_config,
item_type,
public_id,
is_vendor=False): # pragma: no cover
raise click.ClickException(
f"{item_type.title()} {public_id} is already a non-vendor package."
)
configuration = load_item_config(item_type, Path(src))
if public_id.package_version.is_latest:
# get 'concrete' public id, in case it is 'latest'
component_prefix = ComponentType(
item_type), public_id.author, public_id.name
component_id = get_latest_component_id_from_prefix(
ctx.agent_config, component_prefix)
# component id is necessarily found, due to the checks above.
public_id = cast(ComponentId, component_id).public_id
package_id = PackageId(PackageType(item_type), public_id)
click.echo(
f"Ejecting item {package_id.package_type.value} {str(package_id.public_id)}"
)
# first, eject all the vendor packages that depend on this
item_remover = ItemRemoveHelper(ctx, ignore_non_vendor=True)
reverse_dependencies = (
item_remover.get_agent_dependencies_with_reverse_dependencies())
reverse_reachable_dependencies = reachable_nodes(reverse_dependencies,
{package_id})
# the reversed topological order of a graph
# is the topological order of the reverse graph.
eject_order = list(
reversed(find_topological_order(reverse_reachable_dependencies)))
eject_order.remove(package_id)
if len(eject_order) > 0 and not quiet:
click.echo(
f"The following vendor packages will be ejected: {eject_order}")
answer = click.confirm("Do you want to proceed?")
if not answer:
click.echo("Aborted.")
return
for dependency_package_id in eject_order:
# 'dependency_package_id' depends on 'package_id',
# so we need to eject it first
_eject_item(
ctx,
dependency_package_id.package_type.value,
dependency_package_id.public_id,
quiet=True,
)
# copy the vendor package into the non-vendor packages
ctx.clean_paths.append(dst)
copy_package_directory(Path(src), dst)
new_public_id = PublicId(cli_author, public_id.name, DEFAULT_VERSION)
current_version = Version(aea.__version__)
new_aea_range = (
configuration.aea_version
if configuration.aea_version_specifiers.contains(current_version) else
compute_specifier_from_version(current_version))
update_item_config(
item_type,
Path(dst),
author=new_public_id.author,
version=new_public_id.version,
aea_version=new_aea_range,
)
update_item_public_id_in_init(item_type, Path(dst), new_public_id)
shutil.rmtree(src)
# update references in all the other packages
component_type = ComponentType(item_type_plural[:-1])
old_component_id = ComponentId(component_type, public_id)
new_component_id = ComponentId(component_type, new_public_id)
update_references(ctx, {old_component_id: new_component_id})
# need to reload agent configuration with the updated references
try_to_load_agent_config(ctx)
# replace import statements in all the non-vendor packages
replace_all_import_statements(Path(ctx.cwd), ComponentType(item_type),
public_id, new_public_id)
# fingerprint all (non-vendor) packages
fingerprint_all(ctx)
if with_symlinks:
click.echo(
"Adding symlinks from vendor to non-vendor and packages to vendor folders."
)
create_symlink_vendor_to_local(ctx, item_type, new_public_id)
create_symlink_packages_to_vendor(ctx)
click.echo(
f"Successfully ejected {item_type} {public_id} to {dst} as {new_public_id}."
)
def from_configuration_type(
cls, configuration_type: Union[PackageType, str]
) -> "ConfigLoader":
"""Get the configuration loader from the type."""
configuration_type = PackageType(configuration_type)
return ConfigLoaders.from_package_type(configuration_type)
def from_package_type(
cls, configuration_type: Union[PackageType, str]
) -> "ConfigLoader":
configuration_type = PackageType(configuration_type)
return cls._from_configuration_type_to_loaders[configuration_type]
def extract_package_id(match: Match) -> PackageId:
package_type, package = match.group(1), match.group(2)
package_id = PackageId(PackageType(package_type),
PublicId.from_str(package))
return package_id
def _add_item(click_context, item_type, item_public_id) -> None:
"""
Add an item.
:param click_context: the click context.
:param item_type: the item type.
:param item_public_id: the item public id.
:return: None
"""
ctx = cast(Context, click_context.obj)
agent_name = cast(str, ctx.agent_config.agent_name)
item_type_plural = item_type + "s"
supported_items = getattr(ctx.agent_config, item_type_plural)
is_local = ctx.config.get("is_local")
click.echo(
"Adding {} '{}' to the agent '{}'...".format(
item_type, item_public_id, agent_name
)
)
# check if we already have an item with the same name
logger.debug(
"{} already supported by the agent: {}".format(
item_type_plural.capitalize(), supported_items
)
)
if _is_item_present(item_type, item_public_id, ctx):
raise click.ClickException(
"A {} with id '{}/{}' already exists. Aborting...".format(
item_type, item_public_id.author, item_public_id.name
)
)
# find and add protocol
dest_path = get_package_dest_path(
ctx, item_public_id.author, item_type_plural, item_public_id.name
)
ctx.clean_paths.append(dest_path)
if item_public_id in [DEFAULT_CONNECTION, DEFAULT_PROTOCOL, DEFAULT_SKILL]:
source_path = find_item_in_distribution(ctx, item_type, item_public_id)
package_path = copy_package_directory(
ctx,
source_path,
item_type,
item_public_id.name,
item_public_id.author,
dest_path,
)
elif is_local:
source_path = find_item_locally(ctx, item_type, item_public_id)
package_path = copy_package_directory(
ctx,
source_path,
item_type,
item_public_id.name,
item_public_id.author,
dest_path,
)
else:
package_path = fetch_package(
item_type, public_id=item_public_id, cwd=ctx.cwd, dest=dest_path
)
configuration_file_name = _get_default_configuration_file_name_from_type(item_type)
configuration_path = package_path / configuration_file_name
configuration_loader = ConfigLoader.from_configuration_type(PackageType(item_type))
item_configuration = configuration_loader.load(configuration_path.open())
_validate_fingerprint(package_path, item_configuration)
if item_type in {"connection", "skill"}:
_add_protocols(click_context, item_configuration.protocols)
if item_type == "skill":
for contract_public_id in item_configuration.contracts:
if contract_public_id not in ctx.agent_config.contracts:
_add_item(click_context, "contract", contract_public_id)
# add the item to the configurations.
logger.debug(
"Registering the {} into {}".format(item_type, DEFAULT_AEA_CONFIG_FILE)
)
supported_items.add(item_public_id)
ctx.agent_loader.dump(
ctx.agent_config, open(os.path.join(ctx.cwd, DEFAULT_AEA_CONFIG_FILE), "w")
)
