def exception_handler(self, exception: Exception,
function: Callable) -> bool:
"""
Handle exception raised during agent main loop execution.
:param exception: exception raised
:param function: a callable exception raised in.
:return: bool, propagate exception if True otherwise skip it.
"""
# docstyle: ignore # noqa: E800
def log_exception(e, fn):
self.logger.exception(f"<{e}> raised during `{fn}`")
if self._skills_exception_policy == ExceptionPolicyEnum.propagate:
return True
if self._skills_exception_policy == ExceptionPolicyEnum.stop_and_exit:
log_exception(exception, function)
raise _StopRuntime(
AEAException(
f"AEA was terminated cause exception `{exception}` in skills {function}! Please check logs."
))
if self._skills_exception_policy == ExceptionPolicyEnum.just_log:
log_exception(exception, function)
return False
raise AEAException(
f"Unsupported exception policy: {self._skills_exception_policy}")
def _run_agent(agent_dir: Union[PathLike, str],
stop_event: Event,
log_level: Optional[str] = None) -> None:
"""
Load and run agent in a dedicated process.
:param agent_dir: agent configuration directory
:param stop_event: multithreading Event to stop agent run.
:param log_level: debug level applied for AEA in subprocess
:return: None
"""
import asyncio # pylint: disable=import-outside-toplevel
import select # pylint: disable=import-outside-toplevel
import selectors # pylint: disable=import-outside-toplevel
if hasattr(select, "kqueue"): # pragma: nocover # cause platform specific
selector = selectors.SelectSelector()
loop = asyncio.SelectorEventLoop(selector) # type: ignore
asyncio.set_event_loop(loop)
_set_logger(log_level=log_level)
agent = load_agent(agent_dir)
def stop_event_thread() -> None:
try:
stop_event.wait()
except (KeyboardInterrupt, EOFError,
BrokenPipeError) as e: # pragma: nocover
_default_logger.debug(
f"Exception raised in stop_event_thread {e} {type(e)}. Skip it, looks process is closed."
)
finally:
_default_logger.debug(
"_run_agent: stop event raised. call agent.stop")
agent.runtime.stop()
Thread(target=stop_event_thread, daemon=True).start()
try:
agent.start()
except KeyboardInterrupt: # pragma: nocover
_default_logger.debug("_run_agent: keyboard interrupt")
except BaseException as e: # pragma: nocover
_default_logger.exception("exception in _run_agent")
exc = AEAException(f"Raised {type(e)}({e})")
exc.__traceback__ = e.__traceback__
raise exc
finally:
_default_logger.debug("_run_agent: call agent.stop")
agent.stop()
def _check_go_installed() -> None:
"""Checks if go is installed. Sys.exits if not"""
res = shutil.which("go")
if res is None:
raise AEAException( # pragma: nocover
"Please install go before running the `{} connection. Go is available for download here: https://golang.org/doc/install"
.format(PUBLIC_ID))
def _call_stub(ledger_api: LedgerApi, message: ContractApiMessage,
contract: Contract) -> Optional[Union[bytes, JSONLike]]:
"""Try to call stub methods associated to the contract API request performative."""
try:
method: Callable = getattr(contract, message.performative.value)
if message.performative in [
ContractApiMessage.Performative.GET_STATE,
ContractApiMessage.Performative.GET_RAW_MESSAGE,
ContractApiMessage.Performative.GET_RAW_TRANSACTION,
]:
args, kwargs = (
[ledger_api, message.contract_address],
message.kwargs.body,
)
elif message.performative in [ # pragma: nocover
ContractApiMessage.Performative.GET_DEPLOY_TRANSACTION,
]:
args, kwargs = [ledger_api], message.kwargs.body
else: # pragma: nocover
raise AEAException(
f"Unexpected performative: {message.performative}")
data = method(*args, **kwargs)
return data
except (AttributeError, NotImplementedError):
return None
def _get_spec(self, id: CryptoId, module: Optional[str] = None):
"""Get the crypto spec."""
if module is not None:
try:
importlib.import_module(module)
except ImportError:
raise AEAException(
"A module ({}) was specified for the crypto but was not found, "
"make sure the package is installed with `pip install` before calling `aea.crypto.make()`".format(
module
)
)
if id not in self.specs:
raise AEAException("Crypto not registered with id '{}'.".format(id))
return self.specs[id]
def start(self) -> None:
"""
Start the agent.
Performs the following:
- calls connect() on the multiplexer (unless in debug mode), and
- calls setup(), and
- calls start() on the liveness, and
- enters the agent main loop.
While the liveness of the agent is not stopped it continues to loop over:
- increment the tick,
- call to act(),
- sleep for specified timeout,
- call to react(),
- call to update().
:return: None
"""
was_started = self.runtime.start()
if was_started:
self.runtime.wait_completed(sync=True)
else: # pragma: nocover
raise AEAException("Failed to start runtime! Was it already started?")
def _install_dependency(dependency_name: str, dependency: Dependency):
click.echo("Installing {}...".format(pprint.pformat(dependency_name)))
try:
index = dependency.get("index", None)
git_url = dependency.get("git", None)
revision = dependency.get("ref", "")
version_constraint = dependency.get("version", "")
command = [sys.executable, "-m", "pip", "install"]
if git_url is not None:
command += ["-i", index] if index is not None else []
command += [
"git+" + git_url + "@" + revision + "#egg=" + dependency_name
]
else:
command += ["-i", index] if index is not None else []
command += [dependency_name + version_constraint]
logger.debug("Calling '{}'".format(" ".join(command)))
return_code = _run_install_subprocess(command)
if return_code == 1:
# try a second time
return_code = _run_install_subprocess(command)
assert return_code == 0, "Return code != 0."
except Exception as e:
raise AEAException(
"An error occurred while installing {}, {}: {}".format(
dependency_name, dependency, str(e)))
def _execution_control(
self,
fn: Callable,
component: SkillComponent,
args: Optional[Sequence] = None,
kwargs: Optional[Dict] = None,
) -> Any:
"""
Execute skill function in exception handling environment.
Logs error, stop agent or propagate excepion depends on policy defined.
:param fn: function to call
:param component: skill component function belongs to
:param args: optional sequence of arguments to pass to function on call
:param kwargs: optional dict of keyword arguments to pass to function on call
:return: same as function
"""
# docstyle: ignore
def log_exception(e, fn, component):
logger.exception(
f"<{e}> raised during `{fn}` call of `{component}`")
try:
with ExecTimeoutThreadGuard(self._execution_timeout):
return fn(*(args or []), **(kwargs or {}))
except TimeoutException:
logger.warning(
"`{}` of `{}` was terminated as its execution exceeded the timeout of {} seconds. Please refactor your code!"
.format(fn, component, self._execution_timeout))
except Exception as e: # pylint: disable=broad-except
if self._skills_exception_policy == ExceptionPolicyEnum.propagate:
raise
elif self._skills_exception_policy == ExceptionPolicyEnum.just_log:
log_exception(e, fn, component)
elif self._skills_exception_policy == ExceptionPolicyEnum.stop_and_exit:
log_exception(e, fn, component)
self.stop()
raise AEAException(
f"AEA was terminated cause exception `{e}` in skills {component} {fn}! Please check logs."
)
else:
raise AEAException(
f"Unsupported exception policy: {self._skills_exception_policy}"
)
def check_binary(
binary_name: str,
args: List[str],
version_regex: Pattern,
version_lower_bound: VERSION,
):
"""
Check a binary is accessible from the terminal.
It breaks down in:
1) check if the binary is reachable from the system path;
2) check that the version number is higher or equal than the minimum required version.
:param binary_name: the name of the binary.
:param args: the arguments to provide to the binary to retrieve the version.
:param version_regex: the regex used to extract the version from the output.
:param version_lower_bound: the minimum required version.
:return: None
"""
path = shutil.which(binary_name)
if not path:
raise AEAException(
ERROR_MESSAGE_TEMPLATE_BINARY_NOT_FOUND.format(
command=binary_name))
version_getter_command = [binary_name, *args]
stdout = subprocess.check_output(version_getter_command).decode(
"utf-8") # nosec
version_match = version_regex.search(stdout)
if version_match is None:
print(
f"Warning: cannot parse '{binary_name}' version from command: {version_getter_command}. stdout: {stdout}"
)
return
actual_version: VERSION = get_version(
*map(int, version_match.groups(default="0")))
if actual_version < version_lower_bound:
raise AEAException(
ERROR_MESSAGE_TEMPLATE_VERSION_TOO_LOW.format(
command=binary_name,
lower_bound=version_to_string(version_lower_bound),
actual_version=version_to_string(actual_version),
))
print_ok_message(binary_name, actual_version, version_lower_bound)
def _check_go_installed() -> None:
"""Checks if go is installed. Sys.exits if not"""
res = shutil.which("go")
if res is None:
raise AEAException( # pragma: nocover
"Please install go before running the `fetchai/p2p_libp2p:0.1.0` connection. "
"Go is available for download here: https://golang.org/doc/install"
)
def _validate_and_call_callable(api: LedgerApi,
message: ContractApiMessage,
contract: Contract):
"""
Validate a Contract callable, given the performative.
In particular:
- if the performative is either 'get_state' or 'get_raw_transaction', the signature
must accept ledger api as first argument and contract address as second argument,
plus keyword arguments.
- if the performative is either 'get_deploy_transaction' or 'get_raw_message', the signature
must accept ledger api as first argument, plus keyword arguments.
:param api: the ledger api object.
:param message: the contract api request.
:param contract: the contract instance.
:return: the data generated by the method.
"""
try:
method_to_call = getattr(contract, message.callable)
except AttributeError:
raise AEAException(
f"Cannot find {message.callable} in contract {type(contract)}")
full_args_spec = inspect.getfullargspec(method_to_call)
if message.performative in [
ContractApiMessage.Performative.GET_STATE,
ContractApiMessage.Performative.GET_RAW_MESSAGE,
ContractApiMessage.Performative.GET_RAW_TRANSACTION,
]:
if len(full_args_spec.args) < 2:
raise AEAException(
f"Expected two or more positional arguments, got {len(full_args_spec.args)}"
)
return method_to_call(api, message.contract_address,
**message.kwargs.body)
elif message.performative in [
ContractApiMessage.Performative.GET_DEPLOY_TRANSACTION,
]:
if len(full_args_spec.args) < 1:
raise AEAException(
f"Expected one or more positional arguments, got {len(full_args_spec.args)}"
)
return method_to_call(api, **message.kwargs.body)
else: # pragma: nocover
raise AEAException(
f"Unexpected performative: {message.performative}")
def register(self, id: CryptoId, entry_point: EntryPoint, **kwargs):
"""
Register a Crypto module.
:param id: the Cyrpto identifier (e.g. 'fetchai', 'ethereum' etc.)
:param entry_point: the entry point, i.e. 'path.to.module:ClassName'
:return: None
"""
if id in self.specs:
raise AEAException("Cannot re-register id: '{}'".format(id))
self.specs[id] = CryptoSpec(id, entry_point, **kwargs)
def _run_agent(agent_dir: Union[PathLike, str],
stop_event: Event,
log_level: Optional[str] = None) -> None:
"""
Load and run agent in a dedicated process.
:param agent_dir: agent configuration directory
:param stop_event: multithreading Event to stop agent run.
:param log_level: debug level applied for AEA in subprocess
:return: None
"""
_set_logger(log_level=log_level)
agent = load_agent(agent_dir)
def stop_event_thread():
try:
stop_event.wait()
except (KeyboardInterrupt, EOFError,
BrokenPipeError) as e: # pragma: nocover
logger.error(
f"Exception raised in stop_event_thread {e} {type(e)}. Skip it, looks process is closed."
)
finally:
agent.stop()
Thread(target=stop_event_thread, daemon=True).start()
try:
agent.start()
except KeyboardInterrupt: # pragma: nocover
logger.debug("_run_agent: keyboard interrupt")
except BaseException as e: # pragma: nocover
logger.exception("exception in _run_agent")
exc = AEAException(f"Raised {type(e)}({e})")
exc.__traceback__ = e.__traceback__
raise exc
finally:
agent.stop()
def _cast_ctx(context: Union[Context, click.core.Context]) -> Context:
"""
Cast a Context object from context if needed.
:param context: Context or click.core.Context object.
:return: context object.
:raises: AEAException if context is none of Context and click.core.Context types.
"""
if isinstance(context, Context):
return context
if isinstance(context, click.core.Context): # pragma: no cover
return cast(Context, context.obj)
raise AEAException( # pragma: no cover
"clean_after decorator should be used only on methods with Context "
"or click.core.Context object as a first argument.")
def _check_duplicate_classes(name_class_pairs: Sequence[Tuple[str, Type]]):
"""
Given a sequence of pairs (class_name, class_obj), check whether there are duplicates in the class names.
:param name_class_pairs: the sequence of pairs (class_name, class_obj)
:return: None
:raises AEAException: if there are more than one definition of the same class.
"""
names_to_path: Dict[str, str] = {}
for class_name, class_obj in name_class_pairs:
module_path = class_obj.__module__
if class_name in names_to_path:
raise AEAException(
f"Model '{class_name}' present both in {names_to_path[class_name]} and {module_path}. Remove one of them."
)
names_to_path[class_name] = module_path
def start(self) -> None:
"""
Start the agent.
Performs the following:
- calls start() on runtime.
- waits for runtime to complete running (blocking)
:return: None
"""
was_started = self.runtime.start()
if was_started:
self.runtime.wait_completed(sync=True)
else: # pragma: nocover
raise AEAException(
"Failed to start runtime! Was it already started?")
def _install_from_requirement(file: str, install_timeout: float = 300) -> None:
"""
Install from requirements.
:param file: requirement.txt file path
:param install_timeout: timeout to wait pip to install
:return: None
"""
try:
returncode = run_install_subprocess(
[sys.executable, "-m", "pip", "install", "-r", file],
install_timeout)
enforce(returncode == 0, "Return code != 0.")
except Exception:
raise AEAException(
"An error occurred while installing requirement file {}. Stopping..."
.format(file))
def _check_pypi_dependencies(self, configuration: ComponentConfiguration):
"""
Check that PyPI dependencies of a package don't conflict with the existing ones.
:param configuration: the component configuration.
:return: None
:raises AEAException: if some PyPI dependency is conflicting.
"""
all_pypi_dependencies = self._package_dependency_manager.pypi_dependencies
all_pypi_dependencies = merge_dependencies(
all_pypi_dependencies, configuration.pypi_dependencies
)
for pkg_name, dep_info in all_pypi_dependencies.items():
set_specifier = SpecifierSet(dep_info.get("version", ""))
if not is_satisfiable(set_specifier):
raise AEAException(
f"Conflict on package {pkg_name}: specifier set '{dep_info['version']}' not satisfiable."
)
def _check_configuration_not_already_added(
self, configuration: ComponentConfiguration
) -> None:
"""
Check the component configuration has not already been added.
:param configuration: the configuration being added
:return: None
:raises AEAException: if the component is already present.
"""
if (
configuration.component_id
in self._package_dependency_manager.all_dependencies
):
raise AEAException(
"Component '{}' of type '{}' already added.".format(
configuration.public_id, configuration.component_type
)
)
def _check_package_dependencies(
self, configuration: ComponentConfiguration
) -> None:
"""
Check that we have all the dependencies needed to the package.
:return: None
:raises AEAException: if there's a missing dependency.
"""
not_supported_packages = configuration.package_dependencies.difference(
self._package_dependency_manager.all_dependencies
) # type: Set[ComponentId]
has_all_dependencies = len(not_supported_packages) == 0
if not has_all_dependencies:
raise AEAException(
"Package '{}' of type '{}' cannot be added. Missing dependencies: {}".format(
configuration.public_id,
configuration.component_type.value,
pprint.pformat(sorted(map(str, not_supported_packages))),
)
)
def register(
self,
id_: Union[ItemId, str],
entry_point: Union[EntryPoint[ItemType], str],
class_kwargs: Optional[Dict[str, Any]] = None,
**kwargs: Any,
) -> None:
"""
Register an item type.
:param id_: the identifier for the crypto type.
:param entry_point: the entry point to load the crypto object.
:param class_kwargs: keyword arguments to be attached on the class as class variables.
:param kwargs: arguments to provide to the crypto class.
:return: None.
"""
item_id = ItemId(id_)
entry_point = EntryPoint[ItemType](entry_point)
if item_id in self.specs:
raise AEAException("Cannot re-register id: '{}'".format(item_id))
self.specs[item_id] = ItemSpec[ItemType](item_id, entry_point,
class_kwargs, **kwargs)
def handle_dotted_path(value: str) -> Tuple:
"""Separate the path between path to resource and json path to attribute.
Allowed values:
'agent.an_attribute_name'
'protocols.my_protocol.an_attribute_name'
'connections.my_connection.an_attribute_name'
'contracts.my_contract.an_attribute_name'
'skills.my_skill.an_attribute_name'
'vendor.author.[protocols|connections|skills].package_name.attribute_name
:param value: dotted path.
:return: Tuple[list of settings dict keys, filepath, config loader].
"""
parts = value.split(".")
root = parts[0]
if root not in ALLOWED_PATH_ROOTS:
raise AEAException(
"The root of the dotted path must be one of: {}".format(
ALLOWED_PATH_ROOTS))
if (len(parts) < 1 or parts[0] == "agent" and len(parts) < 2
or parts[0] == "vendor" and len(parts) < 5
or parts[0] != "agent" and len(parts) < 3):
raise AEAException(
"The path is too short. Please specify a path up to an attribute name."
)
# if the root is 'agent', stop.
if root == "agent":
resource_type_plural = "agents"
path_to_resource_configuration = Path(DEFAULT_AEA_CONFIG_FILE)
json_path = parts[1:]
elif root == "vendor":
resource_author = parts[1]
resource_type_plural = parts[2]
resource_name = parts[3]
path_to_resource_directory = (Path(".") / "vendor" / resource_author /
resource_type_plural / resource_name)
path_to_resource_configuration = (
path_to_resource_directory /
RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural])
json_path = parts[4:]
if not path_to_resource_directory.exists():
raise AEAException(
"Resource vendor/{}/{}/{} does not exist.".format(
resource_author, resource_type_plural, resource_name))
else:
# navigate the resources of the agent to reach the target configuration file.
resource_type_plural = root
resource_name = parts[1]
path_to_resource_directory = Path(
".") / resource_type_plural / resource_name
path_to_resource_configuration = (
path_to_resource_directory /
RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural])
json_path = parts[2:]
if not path_to_resource_directory.exists():
raise AEAException("Resource {}/{} does not exist.".format(
resource_type_plural, resource_name))
config_loader = ConfigLoader.from_configuration_type(
resource_type_plural[:-1])
return json_path, path_to_resource_configuration, config_loader
def handle_dotted_path(
value: str,
author: str,
aea_project_path: Union[str, Path] = ".",
) -> Tuple[List[str], Path, ConfigLoader, Optional[ComponentId]]:
"""Separate the path between path to resource and json path to attribute.
Allowed values:
'agent.an_attribute_name'
'protocols.my_protocol.an_attribute_name'
'connections.my_connection.an_attribute_name'
'contracts.my_contract.an_attribute_name'
'skills.my_skill.an_attribute_name'
'vendor.author.[protocols|contracts|connections|skills].package_name.attribute_name
We also return the component id to retrieve the configuration of a specific
component. Notice that at this point we don't know the version,
so we put 'latest' as version, but later we will ignore it because
we will filter with only the component prefix (i.e. the triple type, author and name).
:param value: dotted path.
:param author: the author string.
:param aea_project_path: project path
:return: Tuple[list of settings dict keys, filepath, config loader, component id].
"""
parts = value.split(".")
aea_project_path = Path(aea_project_path)
root = parts[0]
if root not in ALLOWED_PATH_ROOTS:
raise AEAException(
"The root of the dotted path must be one of: {}".format(
ALLOWED_PATH_ROOTS))
if (len(parts) < 2 or parts[0] == AGENT and len(parts) < 2
or parts[0] == VENDOR and len(parts) < 5
or parts[0] != AGENT and len(parts) < 3):
raise AEAException(
"The path is too short. Please specify a path up to an attribute name."
)
# if the root is 'agent', stop.
if root == AGENT:
resource_type_plural = AGENTS
path_to_resource_configuration = Path(DEFAULT_AEA_CONFIG_FILE)
json_path = parts[1:]
component_id = None
elif root == VENDOR:
# parse json path
resource_author = parts[1]
resource_type_plural = parts[2]
resource_name = parts[3]
# extract component id
resource_type_singular = resource_type_plural[:-1]
try:
component_type = ComponentType(resource_type_singular)
except ValueError as e:
raise AEAException(
f"'{resource_type_plural}' is not a valid component type. Please use one of {ComponentType.plurals()}."
) from e
component_id = ComponentId(component_type,
PublicId(resource_author, resource_name))
# find path to the resource directory
path_to_resource_directory = (aea_project_path / VENDOR /
resource_author / resource_type_plural /
resource_name)
path_to_resource_configuration = (
path_to_resource_directory /
RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural])
json_path = parts[4:]
if not path_to_resource_directory.exists():
raise AEAException( # pragma: nocover
"Resource vendor/{}/{}/{} does not exist.".format(
resource_author, resource_type_plural, resource_name))
else:
# navigate the resources of the agent to reach the target configuration file.
resource_type_plural = root
resource_name = parts[1]
# extract component id
resource_type_singular = resource_type_plural[:-1]
component_type = ComponentType(resource_type_singular)
resource_author = author
component_id = ComponentId(component_type,
PublicId(resource_author, resource_name))
# find path to the resource directory
path_to_resource_directory = Path(
".") / resource_type_plural / resource_name
path_to_resource_configuration = (
path_to_resource_directory /
RESOURCE_TYPE_TO_CONFIG_FILE[resource_type_plural])
json_path = parts[2:]
if not path_to_resource_directory.exists():
raise AEAException("Resource {}/{} does not exist.".format(
resource_type_plural, resource_name))
config_loader = ConfigLoader.from_configuration_type(
resource_type_plural[:-1])
return json_path, path_to_resource_configuration, config_loader, component_id
def _handle_malformed_string(class_name: str, malformed_id: str):
raise AEAException(
"Malformed {}: '{}'. It must be of the form '{}'.".format(
class_name, malformed_id, CryptoId.REGEX.pattern
)
)
def _find_import_order(
self,
skill_ids: List[ComponentId],
aea_project_path: Path,
skip_consistency_check: bool,
) -> List[ComponentId]:
"""Find import order for skills.
We need to handle skills separately, since skills can depend on each other.
That is, we need to:
- load the skill configurations to find the import order
- detect if there are cycles
- import skills from the leaves of the dependency graph, by finding a topological ordering.
"""
# the adjacency list for the dependency graph
depends_on: Dict[ComponentId, Set[ComponentId]] = defaultdict(set)
# the adjacency list for the inverse dependency graph
supports: Dict[ComponentId, Set[ComponentId]] = defaultdict(set)
# nodes with no incoming edges
roots = copy(skill_ids)
for skill_id in skill_ids:
component_path = self._find_component_directory_from_component_id(
aea_project_path, skill_id
)
configuration = cast(
SkillConfig,
ComponentConfiguration.load(
skill_id.component_type, component_path, skip_consistency_check
),
)
if len(configuration.skills) != 0:
roots.remove(skill_id)
depends_on[skill_id].update(
[
ComponentId(ComponentType.SKILL, skill)
for skill in configuration.skills
]
)
for dependency in configuration.skills:
supports[ComponentId(ComponentType.SKILL, dependency)].add(skill_id)
# find topological order (Kahn's algorithm)
queue: Deque[ComponentId] = deque()
order = []
queue.extend(roots)
while len(queue) > 0:
current = queue.pop()
order.append(current)
for node in supports[
current
]: # pragma: nocover # TODO: extract method and test properly
depends_on[node].discard(current)
if len(depends_on[node]) == 0:
queue.append(node)
if any(len(edges) > 0 for edges in depends_on.values()):
raise AEAException("Cannot load skills, there is a cyclic dependency.")
return order