def test_load_module():
"""Test load module from filepath and dotted notation."""
load_module(
"packages.fetchai.connections.gym.connection",
Path(ROOT_DIR) / "packages" / "fetchai" / "connections" / "gym" /
"connection.py",
)
def from_dir(cls, directory: str) -> "Protocol":
"""
Load a protocol from a directory.
:param directory: the skill directory.
:param agent_context: the agent's context
:return: the Protocol object.
:raises Exception: if the parsing failed.
"""
# check if there is the config file. If not, then return None.
protocol_loader = ConfigLoader("protocol-config_schema.json",
ProtocolConfig)
protocol_config = protocol_loader.load(
open(os.path.join(directory, DEFAULT_PROTOCOL_CONFIG_FILE)))
protocol_module = load_module("protocols",
Path(directory, "serialization.py"))
add_agent_component_module_to_sys_modules("protocol",
protocol_config.name,
protocol_config.author,
protocol_module)
classes = inspect.getmembers(protocol_module, inspect.isclass)
serializer_classes = list(
filter(lambda x: re.match("\\w+Serializer", x[0]), classes))
assert len(
serializer_classes) == 1, "Not exactly one serializer detected."
serializer_class = serializer_classes[0][1]
protocol_id = PublicId(protocol_config.author, protocol_config.name,
protocol_config.version)
protocol = Protocol(protocol_id, serializer_class(), protocol_config)
return protocol
def from_config(cls, configuration: ContractConfig) -> "Contract":
"""
Load contract from configuration
:param configuration: the contract configuration.
:return: the contract object.
"""
assert (
configuration.directory is not None
), "Configuration must be associated with a directory."
directory = configuration.directory
package_modules = load_all_modules(
directory, glob="__init__.py", prefix=configuration.prefix_import_path
)
add_modules_to_sys_modules(package_modules)
contract_module = load_module("contracts", directory / "contract.py")
classes = inspect.getmembers(contract_module, inspect.isclass)
contract_class_name = cast(str, configuration.class_name)
contract_classes = list(
filter(lambda x: re.match(contract_class_name, x[0]), classes)
)
name_to_class = dict(contract_classes)
logger.debug("Processing contract {}".format(contract_class_name))
contract_class = name_to_class.get(contract_class_name, None)
assert contract_class_name is not None, "Contract class '{}' not found.".format(
contract_class_name
)
path = Path(directory, configuration.path_to_contract_interface)
with open(path, "r") as interface_file:
contract_interface = json.load(interface_file)
return contract_class(configuration, contract_interface)
def from_config(cls, configuration: ContractConfig,
**kwargs) -> "Contract":
"""
Load contract from configuration.
:param configuration: the contract configuration.
:return: the contract object.
"""
assert (
configuration.directory
is not None), "Configuration must be associated with a directory."
directory = configuration.directory
load_aea_package(configuration)
contract_module = load_module("contracts", directory / "contract.py")
classes = inspect.getmembers(contract_module, inspect.isclass)
contract_class_name = cast(str, configuration.class_name)
contract_classes = list(
filter(lambda x: re.match(contract_class_name, x[0]), classes))
name_to_class = dict(contract_classes)
logger.debug("Processing contract {}".format(contract_class_name))
contract_class = name_to_class.get(contract_class_name, None)
assert contract_class_name is not None, "Contract class '{}' not found.".format(
contract_class_name)
# TODO: load interfaces here
# contract_interface = configuration.contract_interfaces
return contract_class(configuration, **kwargs)
def parse_module(
cls,
path: str,
handler_configs: Dict[str, SkillComponentConfiguration],
skill_context: SkillContext,
) -> Dict[str, "Handler"]:
"""
Parse the handler module.
:param path: path to the Python module containing the Handler class.
:param handler_configs: the list of handler configurations.
:param skill_context: the skill context
:return: an handler, or None if the parsing fails.
"""
handlers = {} # type: Dict[str, "Handler"]
if handler_configs == {}:
return handlers
handler_names = set(config.class_name for _, config in handler_configs.items())
handler_module = load_module("handlers", Path(path))
classes = inspect.getmembers(handler_module, inspect.isclass)
handler_classes = list(
filter(
lambda x: any(re.match(handler, x[0]) for handler in handler_names)
and not str.startswith(x[1].__module__, "aea.")
and not str.startswith(
x[1].__module__,
f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}",
),
classes,
)
)
name_to_class = dict(handler_classes)
_print_warning_message_for_non_declared_skill_components(
set(name_to_class.keys()),
{handler_config.class_name for handler_config in handler_configs.values()},
"handlers",
path,
)
for handler_id, handler_config in handler_configs.items():
handler_class_name = cast(str, handler_config.class_name)
logger.debug("Processing handler {}".format(handler_class_name))
assert handler_id.isidentifier(), "'{}' is not a valid identifier.".format(
handler_id
)
handler_class = name_to_class.get(handler_class_name, None)
if handler_class is None:
logger.warning(
"Handler '{}' cannot be found.".format(handler_class_name)
)
else:
handler = handler_class(
name=handler_id,
configuration=handler_config,
skill_context=skill_context,
**dict(handler_config.args),
)
handlers[handler_id] = handler
return handlers
def from_config(cls, configuration: ContractConfig,
**kwargs) -> "Contract":
"""
Load contract from configuration.
:param configuration: the contract configuration.
:return: the contract object.
"""
if configuration.directory is None: # pragma: nocover
raise ValueError(
"Configuration must be associated with a directory.")
directory = configuration.directory
load_aea_package(configuration)
contract_module = load_module(CONTRACTS, directory / "contract.py")
classes = inspect.getmembers(contract_module, inspect.isclass)
contract_class_name = cast(str, configuration.class_name)
contract_classes = list(
filter(lambda x: re.match(contract_class_name, x[0]), classes))
name_to_class = dict(contract_classes)
_default_logger.debug(f"Processing contract {contract_class_name}")
contract_class = name_to_class.get(contract_class_name, None)
if contract_class is None:
raise AEAComponentLoadException(
f"Contract class '{contract_class_name}' not found.")
_try_to_register_contract(configuration)
contract = contract_registry.make(str(configuration.public_id),
**kwargs)
return contract
def from_config(cls, configuration: ProtocolConfig) -> "Protocol":
"""
Load the protocol from configuration.
:param configuration: the protocol configuration.
:return: the protocol object.
"""
assert (
configuration.directory
is not None), "Configuration must be associated with a directory."
directory = configuration.directory
package_modules = load_all_modules(
directory,
glob="__init__.py",
prefix=configuration.prefix_import_path)
add_modules_to_sys_modules(package_modules)
serialization_module = load_module("serialization",
Path(directory, "serialization.py"))
classes = inspect.getmembers(serialization_module, inspect.isclass)
serializer_classes = list(
filter(lambda x: re.match("\\w+Serializer", x[0]), classes))
assert len(
serializer_classes) == 1, "Not exactly one serializer detected."
serializer_class = serializer_classes[0][1]
serializer = serializer_class()
return Protocol(configuration, serializer)
def from_config(cls, configuration: ConnectionConfig, identity: Identity,
crypto_store: CryptoStore, **kwargs) -> "Connection":
"""
Load a connection from a configuration.
:param configuration: the connection configuration.
:param identity: the identity object.
:param crypto_store: object to access the connection crypto objects.
:return: an instance of the concrete connection class.
"""
configuration = cast(ConnectionConfig, configuration)
directory = cast(Path, configuration.directory)
load_aea_package(configuration)
connection_module_path = directory / "connection.py"
assert (
connection_module_path.exists()
and connection_module_path.is_file()
), "Connection module '{}' not found.".format(connection_module_path)
connection_module = load_module("connection_module",
directory / "connection.py")
classes = inspect.getmembers(connection_module, inspect.isclass)
connection_class_name = cast(str, configuration.class_name)
connection_classes = list(
filter(lambda x: re.match(connection_class_name, x[0]), classes))
name_to_class = dict(connection_classes)
logger.debug("Processing connection {}".format(connection_class_name))
connection_class = name_to_class.get(connection_class_name, None)
assert connection_class is not None, "Connection class '{}' not found.".format(
connection_class_name)
return connection_class(configuration=configuration,
identity=identity,
crypto_store=crypto_store,
**kwargs)
def set_decision_maker_handler(
self, decision_maker_handler_dotted_path: str, file_path: Path
) -> "AEABuilder":
"""
Set decision maker handler class.
:param decision_maker_handler_dotted_path: the dotted path to the decision maker handler
:param file_path: the file path to the file which contains the decision maker handler
:return: self
"""
dotted_path, class_name = decision_maker_handler_dotted_path.split(":")
module = load_module(dotted_path, file_path)
try:
_class = getattr(module, class_name)
self._decision_maker_handler_class = _class
except Exception as e: # pragma: nocover
logger.error(
"Could not locate decision maker handler for dotted path '{}', class name '{}' and file path '{}'. Error message: {}".format(
dotted_path, class_name, file_path, e
)
)
raise # log and re-raise because we should not build an agent from an. invalid configuration
return self
def parse_module(
cls,
path: str,
behaviour_configs: Dict[str, BehaviourConfig],
skill_context: SkillContext,
) -> Dict[str, "Behaviour"]:
"""
Parse the behaviours module.
:param path: path to the Python module containing the Behaviour classes.
:param behaviour_configs: a list of behaviour configurations.
:param skill_context: the skill context
:return: a list of Behaviour.
"""
behaviours = {} # type: Dict[str, "Behaviour"]
if behaviour_configs == {}:
return behaviours
behaviour_module = load_module("behaviours", Path(path))
classes = inspect.getmembers(behaviour_module, inspect.isclass)
behaviours_classes = list(
filter(
lambda x: re.match("\\w+Behaviour", x[0]) and not str.
startswith(x[1].__module__, "aea."),
classes,
))
name_to_class = dict(behaviours_classes)
_print_warning_message_for_non_declared_skill_components(
set(name_to_class.keys()),
set([
behaviour_config.class_name
for behaviour_config in behaviour_configs.values()
]),
"behaviours",
path,
)
for behaviour_id, behaviour_config in behaviour_configs.items():
behaviour_class_name = cast(str, behaviour_config.class_name)
logger.debug(
"Processing behaviour {}".format(behaviour_class_name))
assert (behaviour_id.isidentifier()
), "'{}' is not a valid identifier.".format(behaviour_id)
behaviour_class = name_to_class.get(behaviour_class_name, None)
if behaviour_class is None:
logger.warning("Behaviour '{}' cannot be found.".format(
behaviour_class_name))
else:
args = behaviour_config.args
assert (
"skill_context" not in args.keys()
), "'skill_context' is a reserved key. Please rename your arguments!"
args["skill_context"] = skill_context
args["name"] = behaviour_id
behaviour = behaviour_class(**args)
behaviours[behaviour_id] = behaviour
return behaviours
def from_config(
cls,
configuration: ConnectionConfig,
identity: Identity,
crypto_store: CryptoStore,
data_dir: str,
**kwargs: Any,
) -> "Connection":
"""
Load a connection from a configuration.
:param configuration: the connection configuration.
:param identity: the identity object.
:param crypto_store: object to access the connection crypto objects.
:param data_dir: the directory of the AEA project data.
:return: an instance of the concrete connection class.
"""
configuration = cast(ConnectionConfig, configuration)
directory = cast(Path, configuration.directory)
load_aea_package(configuration)
connection_module_path = directory / "connection.py"
if not (connection_module_path.exists() and connection_module_path.is_file()):
raise AEAComponentLoadException(
"Connection module '{}' not found.".format(connection_module_path)
)
connection_module = load_module(
"connection_module", directory / "connection.py"
)
classes = inspect.getmembers(connection_module, inspect.isclass)
connection_class_name = cast(str, configuration.class_name)
connection_classes = list(
filter(lambda x: re.match(connection_class_name, x[0]), classes)
)
name_to_class = dict(connection_classes)
logger = get_logger(__name__, identity.name)
logger.debug("Processing connection {}".format(connection_class_name))
connection_class = name_to_class.get(connection_class_name, None)
if connection_class is None:
raise AEAComponentLoadException(
"Connection class '{}' not found.".format(connection_class_name)
)
try:
connection = connection_class(
configuration=configuration,
data_dir=data_dir,
identity=identity,
crypto_store=crypto_store,
**kwargs,
)
except Exception as e: # pragma: nocover # pylint: disable=broad-except
e_str = parse_exception(e)
raise AEAInstantiationException(
f"An error occured during instantiation of connection {configuration.public_id}/{configuration.class_name}:\n{e_str}"
)
return connection
def _load_connection(address: Address, configuration: ConnectionConfig) -> Connection:
"""
Load a connection from a directory.
:param address: the connection address.
:param configuration: the connection configuration.
:return: the connection.
"""
try:
directory = cast(Path, configuration.directory)
package_modules = load_all_modules(
directory, glob="__init__.py", prefix=configuration.prefix_import_path
)
add_modules_to_sys_modules(package_modules)
connection_module_path = directory / "connection.py"
assert (
connection_module_path.exists() and connection_module_path.is_file()
), "Connection module '{}' not found.".format(connection_module_path)
connection_module = load_module(
"connection_module", directory / "connection.py"
)
classes = inspect.getmembers(connection_module, inspect.isclass)
connection_class_name = cast(str, configuration.class_name)
connection_classes = list(
filter(lambda x: re.match(connection_class_name, x[0]), classes)
)
name_to_class = dict(connection_classes)
logger.debug("Processing connection {}".format(connection_class_name))
connection_class = name_to_class.get(connection_class_name, None)
assert connection_class is not None, "Connection class '{}' not found.".format(
connection_class_name
)
return connection_class.from_config(
address=address, configuration=configuration
)
except Exception as e:
raise Exception(
"An error occurred while loading connection {}: {}".format(
configuration.public_id, str(e)
)
)
def parse_module( # pylint: disable=arguments-differ
cls,
path: str,
model_configs: Dict[str, SkillComponentConfiguration],
skill_context: SkillContext,
) -> Dict[str, "Model"]:
"""
Parse the tasks module.
:param path: path to the Python skill module.
:param model_configs: a list of model configurations.
:param skill_context: the skill context
:return: a list of Model.
"""
instances = {} # type: Dict[str, "Model"]
if model_configs == {}:
return instances
models = []
model_names = set(config.class_name for _, config in model_configs.items())
# get all Python modules except the standard ones
ignore_regex = "|".join(["handlers.py", "behaviours.py", "tasks.py", "__.*"])
all_python_modules = Path(path).glob("*.py")
module_paths = set(
map(
str,
filter(
lambda x: not re.match(ignore_regex, x.name), all_python_modules
),
)
)
for module_path in module_paths:
skill_context.logger.debug("Trying to load module {}".format(module_path))
module_name = module_path.replace(".py", "")
model_module = load_module(module_name, Path(module_path))
classes = inspect.getmembers(model_module, inspect.isclass)
filtered_classes = list(
filter(
lambda x: any(re.match(model, x[0]) for model in model_names)
and issubclass(x[1], Model)
and not str.startswith(x[1].__module__, "aea.")
and not str.startswith(
x[1].__module__,
f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}",
),
classes,
)
)
models.extend(filtered_classes)
_check_duplicate_classes(models)
name_to_class = dict(models)
_print_warning_message_for_non_declared_skill_components(
skill_context,
set(name_to_class.keys()),
{model_config.class_name for model_config in model_configs.values()},
"models",
path,
)
for model_id, model_config in model_configs.items():
model_class_name = model_config.class_name
skill_context.logger.debug(
"Processing model id={}, class={}".format(model_id, model_class_name)
)
if not model_id.isidentifier():
raise AEAComponentLoadException( # pragma: nocover
f"'{model_id}' is not a valid identifier."
)
model = name_to_class.get(model_class_name, None)
if model is None:
skill_context.logger.warning(
"Model '{}' cannot be found.".format(model_class_name)
)
else:
try:
model_instance = model(
name=model_id,
skill_context=skill_context,
configuration=model_config,
**dict(model_config.args),
)
except Exception as e: # pylint: disable=broad-except # pragma: nocover
e_str = parse_exception(e)
raise AEAInstantiationException(
f"An error occured during instantiation of model {skill_context.skill_id}/{model_config.class_name}:\n{e_str}"
)
instances[model_id] = model_instance
setattr(skill_context, model_id, model_instance)
return instances
def parse_module( # pylint: disable=arguments-differ
cls,
path: str,
handler_configs: Dict[str, SkillComponentConfiguration],
skill_context: SkillContext,
) -> Dict[str, "Handler"]:
"""
Parse the handler module.
:param path: path to the Python module containing the Handler class.
:param handler_configs: the list of handler configurations.
:param skill_context: the skill context
:return: an handler, or None if the parsing fails.
"""
handlers = {} # type: Dict[str, "Handler"]
if handler_configs == {}:
return handlers
handler_names = set(config.class_name for _, config in handler_configs.items())
handler_module = load_module("handlers", Path(path))
classes = inspect.getmembers(handler_module, inspect.isclass)
handler_classes = list(
filter(
lambda x: any(re.match(handler, x[0]) for handler in handler_names)
and not str.startswith(x[1].__module__, "aea.")
and not str.startswith(
x[1].__module__,
f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}",
),
classes,
)
)
name_to_class = dict(handler_classes)
_print_warning_message_for_non_declared_skill_components(
skill_context,
set(name_to_class.keys()),
{handler_config.class_name for handler_config in handler_configs.values()},
"handlers",
path,
)
for handler_id, handler_config in handler_configs.items():
handler_class_name = cast(str, handler_config.class_name)
skill_context.logger.debug(
"Processing handler {}".format(handler_class_name)
)
if not handler_id.isidentifier():
raise AEAComponentLoadException( # pragma: nocover
f"'{handler_id}' is not a valid identifier."
)
handler_class = name_to_class.get(handler_class_name, None)
if handler_class is None:
skill_context.logger.warning(
"Handler '{}' cannot be found.".format(handler_class_name)
)
else:
try:
handler = handler_class(
name=handler_id,
configuration=handler_config,
skill_context=skill_context,
**dict(handler_config.args),
)
except Exception as e: # pylint: disable=broad-except # pragma: nocover
e_str = parse_exception(e)
raise AEAInstantiationException(
f"An error occured during instantiation of handler {skill_context.skill_id}/{handler_config.class_name}:\n{e_str}"
)
handlers[handler_id] = handler
return handlers
def parse_module(
cls,
path: str,
behaviour_configs: Dict[str, SkillComponentConfiguration],
skill_context: SkillContext,
) -> Dict[str, "Behaviour"]:
"""
Parse the behaviours module.
:param path: path to the Python module containing the Behaviour classes.
:param behaviour_configs: a list of behaviour configurations.
:param skill_context: the skill context
:return: a list of Behaviour.
"""
behaviours = {} # type: Dict[str, "Behaviour"]
if behaviour_configs == {}:
return behaviours
behaviour_names = set(config.class_name
for _, config in behaviour_configs.items())
behaviour_module = load_module("behaviours", Path(path))
classes = inspect.getmembers(behaviour_module, inspect.isclass)
behaviours_classes = list(
filter(
lambda x: any(
re.match(behaviour, x[0])
for behaviour in behaviour_names) and not str.
startswith(x[1].__module__, "aea.") and not str.startswith(
x[1].__module__,
f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}",
),
classes,
))
name_to_class = dict(behaviours_classes)
_print_warning_message_for_non_declared_skill_components(
set(name_to_class.keys()),
{
behaviour_config.class_name
for behaviour_config in behaviour_configs.values()
},
"behaviours",
path,
)
for behaviour_id, behaviour_config in behaviour_configs.items():
behaviour_class_name = cast(str, behaviour_config.class_name)
logger.debug(
"Processing behaviour {}".format(behaviour_class_name))
assert (behaviour_id.isidentifier()
), "'{}' is not a valid identifier.".format(behaviour_id)
behaviour_class = name_to_class.get(behaviour_class_name, None)
if behaviour_class is None:
logger.warning("Behaviour '{}' cannot be found.".format(
behaviour_class_name))
else:
behaviour = behaviour_class(
name=behaviour_id,
configuration=behaviour_config,
skill_context=skill_context,
**dict(behaviour_config.args),
)
behaviours[behaviour_id] = behaviour
return behaviours
def _setup_connection(connection_public_id: PublicId, address: str,
ctx: Context) -> Connection:
"""
Set up a connection.
:param connection_public_id: the public id of the connection.
:param ctx: the CLI context object.
:param address: the address.
:return: a Connection object.
:raises AEAConfigException: if the connection name provided as argument is not declared in the configuration file,
| or if the connection type is not supported by the framework.
"""
# TODO handle the case when there are multiple connections with the same name
_try_to_load_required_protocols(ctx)
supported_connection_ids = ctx.agent_config.connections
if connection_public_id not in supported_connection_ids:
raise AEAConfigException(
"Connection id '{}' not declared in the configuration file.".
format(connection_public_id))
connection_dir = Path("vendor", connection_public_id.author, "connections",
connection_public_id.name)
if not connection_dir.exists():
connection_dir = Path("connections", connection_public_id.name)
try:
connection_config = ctx.connection_loader.load(
open(connection_dir / DEFAULT_CONNECTION_CONFIG_FILE))
except FileNotFoundError:
raise AEAConfigException(
"Connection config for '{}' not found.".format(
connection_public_id))
connection_package = load_agent_component_package(
"connection",
connection_public_id.name,
connection_config.author,
connection_dir,
)
add_agent_component_module_to_sys_modules(
"connection",
connection_public_id.name,
connection_config.author,
connection_package,
)
try:
connection_module = load_module("connection_module",
connection_dir / "connection.py")
except FileNotFoundError:
raise AEAConfigException(
"Connection '{}' not found.".format(connection_public_id))
classes = inspect.getmembers(connection_module, inspect.isclass)
connection_classes = list(
filter(lambda x: re.match("\\w+Connection", x[0]), classes))
name_to_class = dict(connection_classes)
connection_class_name = cast(str, connection_config.class_name)
logger.debug("Processing connection {}".format(connection_class_name))
connection_class = name_to_class.get(connection_class_name, None)
if connection_class is None:
raise AEAConfigException(
"Connection class '{}' not found.".format(connection_class_name))
connection = connection_class.from_config(address, connection_config)
return connection
def parse_module(
cls,
path: str,
model_configs: Dict[str, ModelConfig],
skill_context: SkillContext,
) -> Dict[str, "Model"]:
"""
Parse the tasks module.
:param path: path to the Python skill module.
:param model_configs: a list of model configurations.
:param skill_context: the skill context
:return: a list of Model.
"""
instances = {} # type: Dict[str, "Model"]
if model_configs == {}:
return instances
models = []
model_names = set(config.class_name
for _, config in model_configs.items())
# get all Python modules except the standard ones
ignore_regex = "|".join(
["handlers.py", "behaviours.py", "tasks.py", "__.*"])
all_python_modules = Path(path).glob("*.py")
module_paths = set(
map(
str,
filter(lambda x: not re.match(ignore_regex, x.name),
all_python_modules),
))
for module_path in module_paths:
logger.debug("Trying to load module {}".format(module_path))
module_name = module_path.replace(".py", "")
model_module = load_module(module_name, Path(module_path))
classes = inspect.getmembers(model_module, inspect.isclass)
filtered_classes = list(
filter(
lambda x: any(
re.match(shared, x[0]) for shared in model_names) and
Model in inspect.getmro(x[1]) and not str.startswith(
x[1].__module__, "aea."),
classes,
))
models.extend(filtered_classes)
name_to_class = dict(models)
_print_warning_message_for_non_declared_skill_components(
set(name_to_class.keys()),
set([
model_config.class_name
for model_config in model_configs.values()
]),
"models",
path,
)
for model_id, model_config in model_configs.items():
model_class_name = model_config.class_name
logger.debug("Processing model id={}, class={}".format(
model_id, model_class_name))
assert model_id.isidentifier(
), "'{}' is not a valid identifier.".format(model_id)
model = name_to_class.get(model_class_name, None)
if model is None:
logger.warning(
"Model '{}' cannot be found.".format(model_class_name))
else:
args = model_config.args
assert (
"skill_context" not in args.keys()
), "'skill_context' is a reserved key. Please rename your arguments!"
args["skill_context"] = skill_context
args["name"] = model_id
model_instance = model(**args)
instances[model_id] = model_instance
setattr(skill_context, model_id, model_instance)
return instances
