def fingerprint_package(package_dir: Path, package_type: Union[str, PackageType]) -> None: """ Fingerprint components of an item. :param ctx: the context. :param item_type: the item type. :param item_public_id: the item public id. :return: None """ package_type = PackageType(package_type) item_type = str(package_type) default_config_file_name = _get_default_configuration_file_name_from_type( item_type) config_loader = ConfigLoader.from_configuration_type(item_type) config_file_path = Path(package_dir, default_config_file_name) config = config_loader.load(open_file(config_file_path)) if not package_dir.exists(): # we only permit non-vendorized packages to be fingerprinted raise ValueError("Package not found at path {}".format(package_dir)) fingerprints_dict = _compute_fingerprint( package_dir, ignore_patterns=config.fingerprint_ignore_patterns ) # type: Dict[str, str] # Load item specification yaml file and add fingerprints config.fingerprint = fingerprints_dict config_loader.dump(config, open_file(config_file_path, "w"))
def is_fingerprint_correct(package_path: Path, item_config) -> bool: """ Validate fingerprint of item before adding. :param package_path: path to a package folder. :param item_config: item configuration. :return: None. """ fingerprint = _compute_fingerprint( package_path, ignore_patterns=item_config.fingerprint_ignore_patterns) return item_config.fingerprint == fingerprint
def _validate_fingerprint(package_path, item_config): """ Validate fingerprint of item before adding. :param package_path: path to a package folder. :param item_config: item configuration. :raises ClickException: if fingerprint is incorrect and removes package_path folder. :return: None. """ fingerprint = _compute_fingerprint( package_path, ignore_patterns=item_config.fingerprint_ignore_patterns ) if item_config.fingerprint != fingerprint: rmtree(package_path) raise click.ClickException("Failed to add an item with incorrect fingerprint.")
def compute_fingerprint( package_path: Path, fingerprint_ignore_patterns: Optional[Collection[str]], client: ipfshttpclient.Client, ) -> Dict[str, str]: """ Compute the fingerprint of a package. :param package_path: path to the package. :param fingerprint_ignore_patterns: filename patterns whose matches will be ignored. :param client: the IPFS Client. It is used to compare our implementation | with the true implementation of IPFS hashing. :return: the fingerprint """ fingerprint = _compute_fingerprint( package_path, ignore_patterns=fingerprint_ignore_patterns, ) assert_hash_consistency(fingerprint, package_path, client) return fingerprint
def fingerprint_item(ctx: Context, item_type: str, item_public_id: PublicId) -> None: """ Fingerprint components of an item. :param ctx: the context. :param item_type: the item type. :param item_public_id: the item public id. :return: None """ item_type_plural = item_type + "s" click.echo( "Fingerprinting {} components of '{}' ...".format(item_type, item_public_id) ) # create fingerprints package_dir = Path(ctx.cwd, item_type_plural, item_public_id.name) try: default_config_file_name = _get_default_configuration_file_name_from_type( item_type ) config_loader = ConfigLoader.from_configuration_type(item_type) config_file_path = Path(package_dir, default_config_file_name) config = config_loader.load(config_file_path.open()) if not package_dir.exists(): # we only permit non-vendorized packages to be fingerprinted raise click.ClickException( "Package not found at path {}".format(package_dir) ) fingerprints_dict = _compute_fingerprint( package_dir, ignore_patterns=config.fingerprint_ignore_patterns ) # type: Dict[str, str] # Load item specification yaml file and add fingerprints config.fingerprint = fingerprints_dict config_loader.dump(config, open(config_file_path, "w")) except Exception as e: raise click.ClickException(str(e))
def ipfs_hashing( package_hashes: Dict[str, str], target_dir: str, package_type: str, package_name: str, ipfs_hash_only: IPFSHashOnly, ): """Hashes a package and its components.""" print("Processing package {} of type {}".format(package_name, package_type)) # load config file to get ignore patterns, dump again immediately to impose ordering if package_type == "agents": config = AgentConfig.from_json( yaml.safe_load(open(Path(target_dir, DEFAULT_AEA_CONFIG_FILE)))) yaml_dump(config.json, open(Path(target_dir, DEFAULT_AEA_CONFIG_FILE), "w")) elif package_type == "connections": config = ConnectionConfig.from_json( yaml.safe_load( open(Path(target_dir, DEFAULT_CONNECTION_CONFIG_FILE)))) yaml_dump(config.json, open(Path(target_dir, DEFAULT_CONNECTION_CONFIG_FILE), "w")) elif package_type == "contracts": config = ContractConfig.from_json( yaml.safe_load(open(Path(target_dir, DEFAULT_CONTRACT_CONFIG_FILE)))) yaml_dump(config.json, open(Path(target_dir, DEFAULT_CONTRACT_CONFIG_FILE), "w")) elif package_type == "protocols": config = ProtocolConfig.from_json( yaml.safe_load(open(Path(target_dir, DEFAULT_PROTOCOL_CONFIG_FILE)))) yaml_dump(config.json, open(Path(target_dir, DEFAULT_PROTOCOL_CONFIG_FILE), "w")) elif package_type == "skills": config = SkillConfig.from_json( yaml.safe_load(open(Path(target_dir, DEFAULT_SKILL_CONFIG_FILE)))) yaml_dump(config.json, open(Path(target_dir, DEFAULT_SKILL_CONFIG_FILE), "w")) config = yaml.safe_load(next(Path(target_dir).glob("*.yaml")).open()) ignore_patterns = config.get("fingerprint_ignore_patterns", []) if package_type != "agents": # hash inner components fingerprints_dict = _compute_fingerprint(Path(target_dir), ignore_patterns) # confirm ipfs only generates same hash: for file_name, ipfs_hash in fingerprints_dict.items(): path = os.path.join(target_dir, file_name) ipfsho_hash = ipfs_hash_only.get(path) if ipfsho_hash != ipfs_hash: print("WARNING, hashes don't match for: {}".format(path)) # update fingerprints file_name = package_type[:-1] + ".yaml" yaml_path = os.path.join(target_dir, file_name) file = open(yaml_path, mode="r") # read all lines at once whole_file = file.read() # close the file file.close() file = open(yaml_path, mode="r") # find and replace # TODO this can be simplified after https://github.com/fetchai/agents-aea/issues/932 existing = "" fingerprint_block = False for line in file: if line.find("fingerprint:") == 0: existing += line fingerprint_block = True elif fingerprint_block: if line.find(" ") == 0: # still inside fingerprint block existing += line else: # fingerprint block has ended break if len(fingerprints_dict) > 0: replacement = "fingerprint:\n" ordered_fingerprints_dict = collections.OrderedDict( sorted(fingerprints_dict.items())) for file_name, ipfs_hash in ordered_fingerprints_dict.items(): replacement += " " + file_name + ": " + ipfs_hash + "\n" else: replacement = "fingerprint: {}\n" whole_file = whole_file.replace(existing, replacement) # close the file file.close() # update fingerprints with open(yaml_path, "w") as f: f.write(whole_file) # hash again to get outer hash (this time all files): # TODO we still need to ignore some files result_list = client.add(target_dir) for result_dict in result_list: if package_name == result_dict["Name"]: key = os.path.join(AUTHOR, package_type, package_name) package_hashes[key] = result_dict["Hash"]