def test_load_module(): """Test load module from filepath and dotted notation.""" load_module( "packages.fetchai.connections.gym.connection", Path(ROOT_DIR) / "packages" / "fetchai" / "connections" / "gym" / "connection.py", )
def from_dir(cls, directory: str) -> "Protocol": """ Load a protocol from a directory. :param directory: the skill directory. :param agent_context: the agent's context :return: the Protocol object. :raises Exception: if the parsing failed. """ # check if there is the config file. If not, then return None. protocol_loader = ConfigLoader("protocol-config_schema.json", ProtocolConfig) protocol_config = protocol_loader.load( open(os.path.join(directory, DEFAULT_PROTOCOL_CONFIG_FILE))) protocol_module = load_module("protocols", Path(directory, "serialization.py")) add_agent_component_module_to_sys_modules("protocol", protocol_config.name, protocol_config.author, protocol_module) classes = inspect.getmembers(protocol_module, inspect.isclass) serializer_classes = list( filter(lambda x: re.match("\\w+Serializer", x[0]), classes)) assert len( serializer_classes) == 1, "Not exactly one serializer detected." serializer_class = serializer_classes[0][1] protocol_id = PublicId(protocol_config.author, protocol_config.name, protocol_config.version) protocol = Protocol(protocol_id, serializer_class(), protocol_config) return protocol
def from_config(cls, configuration: ContractConfig) -> "Contract": """ Load contract from configuration :param configuration: the contract configuration. :return: the contract object. """ assert ( configuration.directory is not None ), "Configuration must be associated with a directory." directory = configuration.directory package_modules = load_all_modules( directory, glob="__init__.py", prefix=configuration.prefix_import_path ) add_modules_to_sys_modules(package_modules) contract_module = load_module("contracts", directory / "contract.py") classes = inspect.getmembers(contract_module, inspect.isclass) contract_class_name = cast(str, configuration.class_name) contract_classes = list( filter(lambda x: re.match(contract_class_name, x[0]), classes) ) name_to_class = dict(contract_classes) logger.debug("Processing contract {}".format(contract_class_name)) contract_class = name_to_class.get(contract_class_name, None) assert contract_class_name is not None, "Contract class '{}' not found.".format( contract_class_name ) path = Path(directory, configuration.path_to_contract_interface) with open(path, "r") as interface_file: contract_interface = json.load(interface_file) return contract_class(configuration, contract_interface)
def from_config(cls, configuration: ContractConfig, **kwargs) -> "Contract": """ Load contract from configuration. :param configuration: the contract configuration. :return: the contract object. """ assert ( configuration.directory is not None), "Configuration must be associated with a directory." directory = configuration.directory load_aea_package(configuration) contract_module = load_module("contracts", directory / "contract.py") classes = inspect.getmembers(contract_module, inspect.isclass) contract_class_name = cast(str, configuration.class_name) contract_classes = list( filter(lambda x: re.match(contract_class_name, x[0]), classes)) name_to_class = dict(contract_classes) logger.debug("Processing contract {}".format(contract_class_name)) contract_class = name_to_class.get(contract_class_name, None) assert contract_class_name is not None, "Contract class '{}' not found.".format( contract_class_name) # TODO: load interfaces here # contract_interface = configuration.contract_interfaces return contract_class(configuration, **kwargs)
def parse_module( cls, path: str, handler_configs: Dict[str, SkillComponentConfiguration], skill_context: SkillContext, ) -> Dict[str, "Handler"]: """ Parse the handler module. :param path: path to the Python module containing the Handler class. :param handler_configs: the list of handler configurations. :param skill_context: the skill context :return: an handler, or None if the parsing fails. """ handlers = {} # type: Dict[str, "Handler"] if handler_configs == {}: return handlers handler_names = set(config.class_name for _, config in handler_configs.items()) handler_module = load_module("handlers", Path(path)) classes = inspect.getmembers(handler_module, inspect.isclass) handler_classes = list( filter( lambda x: any(re.match(handler, x[0]) for handler in handler_names) and not str.startswith(x[1].__module__, "aea.") and not str.startswith( x[1].__module__, f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}", ), classes, ) ) name_to_class = dict(handler_classes) _print_warning_message_for_non_declared_skill_components( set(name_to_class.keys()), {handler_config.class_name for handler_config in handler_configs.values()}, "handlers", path, ) for handler_id, handler_config in handler_configs.items(): handler_class_name = cast(str, handler_config.class_name) logger.debug("Processing handler {}".format(handler_class_name)) assert handler_id.isidentifier(), "'{}' is not a valid identifier.".format( handler_id ) handler_class = name_to_class.get(handler_class_name, None) if handler_class is None: logger.warning( "Handler '{}' cannot be found.".format(handler_class_name) ) else: handler = handler_class( name=handler_id, configuration=handler_config, skill_context=skill_context, **dict(handler_config.args), ) handlers[handler_id] = handler return handlers
def from_config(cls, configuration: ContractConfig, **kwargs) -> "Contract": """ Load contract from configuration. :param configuration: the contract configuration. :return: the contract object. """ if configuration.directory is None: # pragma: nocover raise ValueError( "Configuration must be associated with a directory.") directory = configuration.directory load_aea_package(configuration) contract_module = load_module(CONTRACTS, directory / "contract.py") classes = inspect.getmembers(contract_module, inspect.isclass) contract_class_name = cast(str, configuration.class_name) contract_classes = list( filter(lambda x: re.match(contract_class_name, x[0]), classes)) name_to_class = dict(contract_classes) _default_logger.debug(f"Processing contract {contract_class_name}") contract_class = name_to_class.get(contract_class_name, None) if contract_class is None: raise AEAComponentLoadException( f"Contract class '{contract_class_name}' not found.") _try_to_register_contract(configuration) contract = contract_registry.make(str(configuration.public_id), **kwargs) return contract
def from_config(cls, configuration: ProtocolConfig) -> "Protocol": """ Load the protocol from configuration. :param configuration: the protocol configuration. :return: the protocol object. """ assert ( configuration.directory is not None), "Configuration must be associated with a directory." directory = configuration.directory package_modules = load_all_modules( directory, glob="__init__.py", prefix=configuration.prefix_import_path) add_modules_to_sys_modules(package_modules) serialization_module = load_module("serialization", Path(directory, "serialization.py")) classes = inspect.getmembers(serialization_module, inspect.isclass) serializer_classes = list( filter(lambda x: re.match("\\w+Serializer", x[0]), classes)) assert len( serializer_classes) == 1, "Not exactly one serializer detected." serializer_class = serializer_classes[0][1] serializer = serializer_class() return Protocol(configuration, serializer)
def from_config(cls, configuration: ConnectionConfig, identity: Identity, crypto_store: CryptoStore, **kwargs) -> "Connection": """ Load a connection from a configuration. :param configuration: the connection configuration. :param identity: the identity object. :param crypto_store: object to access the connection crypto objects. :return: an instance of the concrete connection class. """ configuration = cast(ConnectionConfig, configuration) directory = cast(Path, configuration.directory) load_aea_package(configuration) connection_module_path = directory / "connection.py" assert ( connection_module_path.exists() and connection_module_path.is_file() ), "Connection module '{}' not found.".format(connection_module_path) connection_module = load_module("connection_module", directory / "connection.py") classes = inspect.getmembers(connection_module, inspect.isclass) connection_class_name = cast(str, configuration.class_name) connection_classes = list( filter(lambda x: re.match(connection_class_name, x[0]), classes)) name_to_class = dict(connection_classes) logger.debug("Processing connection {}".format(connection_class_name)) connection_class = name_to_class.get(connection_class_name, None) assert connection_class is not None, "Connection class '{}' not found.".format( connection_class_name) return connection_class(configuration=configuration, identity=identity, crypto_store=crypto_store, **kwargs)
def set_decision_maker_handler( self, decision_maker_handler_dotted_path: str, file_path: Path ) -> "AEABuilder": """ Set decision maker handler class. :param decision_maker_handler_dotted_path: the dotted path to the decision maker handler :param file_path: the file path to the file which contains the decision maker handler :return: self """ dotted_path, class_name = decision_maker_handler_dotted_path.split(":") module = load_module(dotted_path, file_path) try: _class = getattr(module, class_name) self._decision_maker_handler_class = _class except Exception as e: # pragma: nocover logger.error( "Could not locate decision maker handler for dotted path '{}', class name '{}' and file path '{}'. Error message: {}".format( dotted_path, class_name, file_path, e ) ) raise # log and re-raise because we should not build an agent from an. invalid configuration return self
def parse_module( cls, path: str, behaviour_configs: Dict[str, BehaviourConfig], skill_context: SkillContext, ) -> Dict[str, "Behaviour"]: """ Parse the behaviours module. :param path: path to the Python module containing the Behaviour classes. :param behaviour_configs: a list of behaviour configurations. :param skill_context: the skill context :return: a list of Behaviour. """ behaviours = {} # type: Dict[str, "Behaviour"] if behaviour_configs == {}: return behaviours behaviour_module = load_module("behaviours", Path(path)) classes = inspect.getmembers(behaviour_module, inspect.isclass) behaviours_classes = list( filter( lambda x: re.match("\\w+Behaviour", x[0]) and not str. startswith(x[1].__module__, "aea."), classes, )) name_to_class = dict(behaviours_classes) _print_warning_message_for_non_declared_skill_components( set(name_to_class.keys()), set([ behaviour_config.class_name for behaviour_config in behaviour_configs.values() ]), "behaviours", path, ) for behaviour_id, behaviour_config in behaviour_configs.items(): behaviour_class_name = cast(str, behaviour_config.class_name) logger.debug( "Processing behaviour {}".format(behaviour_class_name)) assert (behaviour_id.isidentifier() ), "'{}' is not a valid identifier.".format(behaviour_id) behaviour_class = name_to_class.get(behaviour_class_name, None) if behaviour_class is None: logger.warning("Behaviour '{}' cannot be found.".format( behaviour_class_name)) else: args = behaviour_config.args assert ( "skill_context" not in args.keys() ), "'skill_context' is a reserved key. Please rename your arguments!" args["skill_context"] = skill_context args["name"] = behaviour_id behaviour = behaviour_class(**args) behaviours[behaviour_id] = behaviour return behaviours
def from_config( cls, configuration: ConnectionConfig, identity: Identity, crypto_store: CryptoStore, data_dir: str, **kwargs: Any, ) -> "Connection": """ Load a connection from a configuration. :param configuration: the connection configuration. :param identity: the identity object. :param crypto_store: object to access the connection crypto objects. :param data_dir: the directory of the AEA project data. :return: an instance of the concrete connection class. """ configuration = cast(ConnectionConfig, configuration) directory = cast(Path, configuration.directory) load_aea_package(configuration) connection_module_path = directory / "connection.py" if not (connection_module_path.exists() and connection_module_path.is_file()): raise AEAComponentLoadException( "Connection module '{}' not found.".format(connection_module_path) ) connection_module = load_module( "connection_module", directory / "connection.py" ) classes = inspect.getmembers(connection_module, inspect.isclass) connection_class_name = cast(str, configuration.class_name) connection_classes = list( filter(lambda x: re.match(connection_class_name, x[0]), classes) ) name_to_class = dict(connection_classes) logger = get_logger(__name__, identity.name) logger.debug("Processing connection {}".format(connection_class_name)) connection_class = name_to_class.get(connection_class_name, None) if connection_class is None: raise AEAComponentLoadException( "Connection class '{}' not found.".format(connection_class_name) ) try: connection = connection_class( configuration=configuration, data_dir=data_dir, identity=identity, crypto_store=crypto_store, **kwargs, ) except Exception as e: # pragma: nocover # pylint: disable=broad-except e_str = parse_exception(e) raise AEAInstantiationException( f"An error occured during instantiation of connection {configuration.public_id}/{configuration.class_name}:\n{e_str}" ) return connection
def _load_connection(address: Address, configuration: ConnectionConfig) -> Connection: """ Load a connection from a directory. :param address: the connection address. :param configuration: the connection configuration. :return: the connection. """ try: directory = cast(Path, configuration.directory) package_modules = load_all_modules( directory, glob="__init__.py", prefix=configuration.prefix_import_path ) add_modules_to_sys_modules(package_modules) connection_module_path = directory / "connection.py" assert ( connection_module_path.exists() and connection_module_path.is_file() ), "Connection module '{}' not found.".format(connection_module_path) connection_module = load_module( "connection_module", directory / "connection.py" ) classes = inspect.getmembers(connection_module, inspect.isclass) connection_class_name = cast(str, configuration.class_name) connection_classes = list( filter(lambda x: re.match(connection_class_name, x[0]), classes) ) name_to_class = dict(connection_classes) logger.debug("Processing connection {}".format(connection_class_name)) connection_class = name_to_class.get(connection_class_name, None) assert connection_class is not None, "Connection class '{}' not found.".format( connection_class_name ) return connection_class.from_config( address=address, configuration=configuration ) except Exception as e: raise Exception( "An error occurred while loading connection {}: {}".format( configuration.public_id, str(e) ) )
def parse_module( # pylint: disable=arguments-differ cls, path: str, model_configs: Dict[str, SkillComponentConfiguration], skill_context: SkillContext, ) -> Dict[str, "Model"]: """ Parse the tasks module. :param path: path to the Python skill module. :param model_configs: a list of model configurations. :param skill_context: the skill context :return: a list of Model. """ instances = {} # type: Dict[str, "Model"] if model_configs == {}: return instances models = [] model_names = set(config.class_name for _, config in model_configs.items()) # get all Python modules except the standard ones ignore_regex = "|".join(["handlers.py", "behaviours.py", "tasks.py", "__.*"]) all_python_modules = Path(path).glob("*.py") module_paths = set( map( str, filter( lambda x: not re.match(ignore_regex, x.name), all_python_modules ), ) ) for module_path in module_paths: skill_context.logger.debug("Trying to load module {}".format(module_path)) module_name = module_path.replace(".py", "") model_module = load_module(module_name, Path(module_path)) classes = inspect.getmembers(model_module, inspect.isclass) filtered_classes = list( filter( lambda x: any(re.match(model, x[0]) for model in model_names) and issubclass(x[1], Model) and not str.startswith(x[1].__module__, "aea.") and not str.startswith( x[1].__module__, f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}", ), classes, ) ) models.extend(filtered_classes) _check_duplicate_classes(models) name_to_class = dict(models) _print_warning_message_for_non_declared_skill_components( skill_context, set(name_to_class.keys()), {model_config.class_name for model_config in model_configs.values()}, "models", path, ) for model_id, model_config in model_configs.items(): model_class_name = model_config.class_name skill_context.logger.debug( "Processing model id={}, class={}".format(model_id, model_class_name) ) if not model_id.isidentifier(): raise AEAComponentLoadException( # pragma: nocover f"'{model_id}' is not a valid identifier." ) model = name_to_class.get(model_class_name, None) if model is None: skill_context.logger.warning( "Model '{}' cannot be found.".format(model_class_name) ) else: try: model_instance = model( name=model_id, skill_context=skill_context, configuration=model_config, **dict(model_config.args), ) except Exception as e: # pylint: disable=broad-except # pragma: nocover e_str = parse_exception(e) raise AEAInstantiationException( f"An error occured during instantiation of model {skill_context.skill_id}/{model_config.class_name}:\n{e_str}" ) instances[model_id] = model_instance setattr(skill_context, model_id, model_instance) return instances
def parse_module( # pylint: disable=arguments-differ cls, path: str, handler_configs: Dict[str, SkillComponentConfiguration], skill_context: SkillContext, ) -> Dict[str, "Handler"]: """ Parse the handler module. :param path: path to the Python module containing the Handler class. :param handler_configs: the list of handler configurations. :param skill_context: the skill context :return: an handler, or None if the parsing fails. """ handlers = {} # type: Dict[str, "Handler"] if handler_configs == {}: return handlers handler_names = set(config.class_name for _, config in handler_configs.items()) handler_module = load_module("handlers", Path(path)) classes = inspect.getmembers(handler_module, inspect.isclass) handler_classes = list( filter( lambda x: any(re.match(handler, x[0]) for handler in handler_names) and not str.startswith(x[1].__module__, "aea.") and not str.startswith( x[1].__module__, f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}", ), classes, ) ) name_to_class = dict(handler_classes) _print_warning_message_for_non_declared_skill_components( skill_context, set(name_to_class.keys()), {handler_config.class_name for handler_config in handler_configs.values()}, "handlers", path, ) for handler_id, handler_config in handler_configs.items(): handler_class_name = cast(str, handler_config.class_name) skill_context.logger.debug( "Processing handler {}".format(handler_class_name) ) if not handler_id.isidentifier(): raise AEAComponentLoadException( # pragma: nocover f"'{handler_id}' is not a valid identifier." ) handler_class = name_to_class.get(handler_class_name, None) if handler_class is None: skill_context.logger.warning( "Handler '{}' cannot be found.".format(handler_class_name) ) else: try: handler = handler_class( name=handler_id, configuration=handler_config, skill_context=skill_context, **dict(handler_config.args), ) except Exception as e: # pylint: disable=broad-except # pragma: nocover e_str = parse_exception(e) raise AEAInstantiationException( f"An error occured during instantiation of handler {skill_context.skill_id}/{handler_config.class_name}:\n{e_str}" ) handlers[handler_id] = handler return handlers
def parse_module( cls, path: str, behaviour_configs: Dict[str, SkillComponentConfiguration], skill_context: SkillContext, ) -> Dict[str, "Behaviour"]: """ Parse the behaviours module. :param path: path to the Python module containing the Behaviour classes. :param behaviour_configs: a list of behaviour configurations. :param skill_context: the skill context :return: a list of Behaviour. """ behaviours = {} # type: Dict[str, "Behaviour"] if behaviour_configs == {}: return behaviours behaviour_names = set(config.class_name for _, config in behaviour_configs.items()) behaviour_module = load_module("behaviours", Path(path)) classes = inspect.getmembers(behaviour_module, inspect.isclass) behaviours_classes = list( filter( lambda x: any( re.match(behaviour, x[0]) for behaviour in behaviour_names) and not str. startswith(x[1].__module__, "aea.") and not str.startswith( x[1].__module__, f"packages.{skill_context.skill_id.author}.skills.{skill_context.skill_id.name}", ), classes, )) name_to_class = dict(behaviours_classes) _print_warning_message_for_non_declared_skill_components( set(name_to_class.keys()), { behaviour_config.class_name for behaviour_config in behaviour_configs.values() }, "behaviours", path, ) for behaviour_id, behaviour_config in behaviour_configs.items(): behaviour_class_name = cast(str, behaviour_config.class_name) logger.debug( "Processing behaviour {}".format(behaviour_class_name)) assert (behaviour_id.isidentifier() ), "'{}' is not a valid identifier.".format(behaviour_id) behaviour_class = name_to_class.get(behaviour_class_name, None) if behaviour_class is None: logger.warning("Behaviour '{}' cannot be found.".format( behaviour_class_name)) else: behaviour = behaviour_class( name=behaviour_id, configuration=behaviour_config, skill_context=skill_context, **dict(behaviour_config.args), ) behaviours[behaviour_id] = behaviour return behaviours
def _setup_connection(connection_public_id: PublicId, address: str, ctx: Context) -> Connection: """ Set up a connection. :param connection_public_id: the public id of the connection. :param ctx: the CLI context object. :param address: the address. :return: a Connection object. :raises AEAConfigException: if the connection name provided as argument is not declared in the configuration file, | or if the connection type is not supported by the framework. """ # TODO handle the case when there are multiple connections with the same name _try_to_load_required_protocols(ctx) supported_connection_ids = ctx.agent_config.connections if connection_public_id not in supported_connection_ids: raise AEAConfigException( "Connection id '{}' not declared in the configuration file.". format(connection_public_id)) connection_dir = Path("vendor", connection_public_id.author, "connections", connection_public_id.name) if not connection_dir.exists(): connection_dir = Path("connections", connection_public_id.name) try: connection_config = ctx.connection_loader.load( open(connection_dir / DEFAULT_CONNECTION_CONFIG_FILE)) except FileNotFoundError: raise AEAConfigException( "Connection config for '{}' not found.".format( connection_public_id)) connection_package = load_agent_component_package( "connection", connection_public_id.name, connection_config.author, connection_dir, ) add_agent_component_module_to_sys_modules( "connection", connection_public_id.name, connection_config.author, connection_package, ) try: connection_module = load_module("connection_module", connection_dir / "connection.py") except FileNotFoundError: raise AEAConfigException( "Connection '{}' not found.".format(connection_public_id)) classes = inspect.getmembers(connection_module, inspect.isclass) connection_classes = list( filter(lambda x: re.match("\\w+Connection", x[0]), classes)) name_to_class = dict(connection_classes) connection_class_name = cast(str, connection_config.class_name) logger.debug("Processing connection {}".format(connection_class_name)) connection_class = name_to_class.get(connection_class_name, None) if connection_class is None: raise AEAConfigException( "Connection class '{}' not found.".format(connection_class_name)) connection = connection_class.from_config(address, connection_config) return connection
def parse_module( cls, path: str, model_configs: Dict[str, ModelConfig], skill_context: SkillContext, ) -> Dict[str, "Model"]: """ Parse the tasks module. :param path: path to the Python skill module. :param model_configs: a list of model configurations. :param skill_context: the skill context :return: a list of Model. """ instances = {} # type: Dict[str, "Model"] if model_configs == {}: return instances models = [] model_names = set(config.class_name for _, config in model_configs.items()) # get all Python modules except the standard ones ignore_regex = "|".join( ["handlers.py", "behaviours.py", "tasks.py", "__.*"]) all_python_modules = Path(path).glob("*.py") module_paths = set( map( str, filter(lambda x: not re.match(ignore_regex, x.name), all_python_modules), )) for module_path in module_paths: logger.debug("Trying to load module {}".format(module_path)) module_name = module_path.replace(".py", "") model_module = load_module(module_name, Path(module_path)) classes = inspect.getmembers(model_module, inspect.isclass) filtered_classes = list( filter( lambda x: any( re.match(shared, x[0]) for shared in model_names) and Model in inspect.getmro(x[1]) and not str.startswith( x[1].__module__, "aea."), classes, )) models.extend(filtered_classes) name_to_class = dict(models) _print_warning_message_for_non_declared_skill_components( set(name_to_class.keys()), set([ model_config.class_name for model_config in model_configs.values() ]), "models", path, ) for model_id, model_config in model_configs.items(): model_class_name = model_config.class_name logger.debug("Processing model id={}, class={}".format( model_id, model_class_name)) assert model_id.isidentifier( ), "'{}' is not a valid identifier.".format(model_id) model = name_to_class.get(model_class_name, None) if model is None: logger.warning( "Model '{}' cannot be found.".format(model_class_name)) else: args = model_config.args assert ( "skill_context" not in args.keys() ), "'skill_context' is a reserved key. Please rename your arguments!" args["skill_context"] = skill_context args["name"] = model_id model_instance = model(**args) instances[model_id] = model_instance setattr(skill_context, model_id, model_instance) return instances