def fingerprint_package(package_dir: Path,
package_type: Union[str, PackageType]) -> None:
"""
Fingerprint components of an item.
:param ctx: the context.
:param item_type: the item type.
:param item_public_id: the item public id.
:return: None
"""
package_type = PackageType(package_type)
item_type = str(package_type)
default_config_file_name = _get_default_configuration_file_name_from_type(
item_type)
config_loader = ConfigLoader.from_configuration_type(item_type)
config_file_path = Path(package_dir, default_config_file_name)
config = config_loader.load(open_file(config_file_path))
if not package_dir.exists():
# we only permit non-vendorized packages to be fingerprinted
raise ValueError("Package not found at path {}".format(package_dir))
fingerprints_dict = _compute_fingerprint(
package_dir, ignore_patterns=config.fingerprint_ignore_patterns
) # type: Dict[str, str]
# Load item specification yaml file and add fingerprints
config.fingerprint = fingerprints_dict
config_loader.dump(config, open_file(config_file_path, "w"))
def is_fingerprint_correct(package_path: Path, item_config) -> bool:
"""
Validate fingerprint of item before adding.
:param package_path: path to a package folder.
:param item_config: item configuration.
:return: None.
"""
fingerprint = _compute_fingerprint(
package_path, ignore_patterns=item_config.fingerprint_ignore_patterns)
return item_config.fingerprint == fingerprint
def _validate_fingerprint(package_path, item_config):
"""
Validate fingerprint of item before adding.
:param package_path: path to a package folder.
:param item_config: item configuration.
:raises ClickException: if fingerprint is incorrect and removes package_path folder.
:return: None.
"""
fingerprint = _compute_fingerprint(
package_path, ignore_patterns=item_config.fingerprint_ignore_patterns
)
if item_config.fingerprint != fingerprint:
rmtree(package_path)
raise click.ClickException("Failed to add an item with incorrect fingerprint.")
def compute_fingerprint(
package_path: Path,
fingerprint_ignore_patterns: Optional[Collection[str]],
client: ipfshttpclient.Client,
) -> Dict[str, str]:
"""
Compute the fingerprint of a package.
:param package_path: path to the package.
:param fingerprint_ignore_patterns: filename patterns whose matches will be ignored.
:param client: the IPFS Client. It is used to compare our implementation
| with the true implementation of IPFS hashing.
:return: the fingerprint
"""
fingerprint = _compute_fingerprint(
package_path, ignore_patterns=fingerprint_ignore_patterns,
)
assert_hash_consistency(fingerprint, package_path, client)
return fingerprint
def fingerprint_item(ctx: Context, item_type: str, item_public_id: PublicId) -> None:
"""
Fingerprint components of an item.
:param ctx: the context.
:param item_type: the item type.
:param item_public_id: the item public id.
:return: None
"""
item_type_plural = item_type + "s"
click.echo(
"Fingerprinting {} components of '{}' ...".format(item_type, item_public_id)
)
# create fingerprints
package_dir = Path(ctx.cwd, item_type_plural, item_public_id.name)
try:
default_config_file_name = _get_default_configuration_file_name_from_type(
item_type
)
config_loader = ConfigLoader.from_configuration_type(item_type)
config_file_path = Path(package_dir, default_config_file_name)
config = config_loader.load(config_file_path.open())
if not package_dir.exists():
# we only permit non-vendorized packages to be fingerprinted
raise click.ClickException(
"Package not found at path {}".format(package_dir)
)
fingerprints_dict = _compute_fingerprint(
package_dir, ignore_patterns=config.fingerprint_ignore_patterns
) # type: Dict[str, str]
# Load item specification yaml file and add fingerprints
config.fingerprint = fingerprints_dict
config_loader.dump(config, open(config_file_path, "w"))
except Exception as e:
raise click.ClickException(str(e))
def ipfs_hashing(
package_hashes: Dict[str, str],
target_dir: str,
package_type: str,
package_name: str,
ipfs_hash_only: IPFSHashOnly,
):
"""Hashes a package and its components."""
print("Processing package {} of type {}".format(package_name,
package_type))
# load config file to get ignore patterns, dump again immediately to impose ordering
if package_type == "agents":
config = AgentConfig.from_json(
yaml.safe_load(open(Path(target_dir, DEFAULT_AEA_CONFIG_FILE))))
yaml_dump(config.json,
open(Path(target_dir, DEFAULT_AEA_CONFIG_FILE), "w"))
elif package_type == "connections":
config = ConnectionConfig.from_json(
yaml.safe_load(
open(Path(target_dir, DEFAULT_CONNECTION_CONFIG_FILE))))
yaml_dump(config.json,
open(Path(target_dir, DEFAULT_CONNECTION_CONFIG_FILE), "w"))
elif package_type == "contracts":
config = ContractConfig.from_json(
yaml.safe_load(open(Path(target_dir,
DEFAULT_CONTRACT_CONFIG_FILE))))
yaml_dump(config.json,
open(Path(target_dir, DEFAULT_CONTRACT_CONFIG_FILE), "w"))
elif package_type == "protocols":
config = ProtocolConfig.from_json(
yaml.safe_load(open(Path(target_dir,
DEFAULT_PROTOCOL_CONFIG_FILE))))
yaml_dump(config.json,
open(Path(target_dir, DEFAULT_PROTOCOL_CONFIG_FILE), "w"))
elif package_type == "skills":
config = SkillConfig.from_json(
yaml.safe_load(open(Path(target_dir, DEFAULT_SKILL_CONFIG_FILE))))
yaml_dump(config.json,
open(Path(target_dir, DEFAULT_SKILL_CONFIG_FILE), "w"))
config = yaml.safe_load(next(Path(target_dir).glob("*.yaml")).open())
ignore_patterns = config.get("fingerprint_ignore_patterns", [])
if package_type != "agents":
# hash inner components
fingerprints_dict = _compute_fingerprint(Path(target_dir),
ignore_patterns)
# confirm ipfs only generates same hash:
for file_name, ipfs_hash in fingerprints_dict.items():
path = os.path.join(target_dir, file_name)
ipfsho_hash = ipfs_hash_only.get(path)
if ipfsho_hash != ipfs_hash:
print("WARNING, hashes don't match for: {}".format(path))
# update fingerprints
file_name = package_type[:-1] + ".yaml"
yaml_path = os.path.join(target_dir, file_name)
file = open(yaml_path, mode="r")
# read all lines at once
whole_file = file.read()
# close the file
file.close()
file = open(yaml_path, mode="r")
# find and replace
# TODO this can be simplified after https://github.com/fetchai/agents-aea/issues/932
existing = ""
fingerprint_block = False
for line in file:
if line.find("fingerprint:") == 0:
existing += line
fingerprint_block = True
elif fingerprint_block:
if line.find(" ") == 0:
# still inside fingerprint block
existing += line
else:
# fingerprint block has ended
break
if len(fingerprints_dict) > 0:
replacement = "fingerprint:\n"
ordered_fingerprints_dict = collections.OrderedDict(
sorted(fingerprints_dict.items()))
for file_name, ipfs_hash in ordered_fingerprints_dict.items():
replacement += " " + file_name + ": " + ipfs_hash + "\n"
else:
replacement = "fingerprint: {}\n"
whole_file = whole_file.replace(existing, replacement)
# close the file
file.close()
# update fingerprints
with open(yaml_path, "w") as f:
f.write(whole_file)
# hash again to get outer hash (this time all files):
# TODO we still need to ignore some files
result_list = client.add(target_dir)
for result_dict in result_list:
if package_name == result_dict["Name"]:
key = os.path.join(AUTHOR, package_type, package_name)
package_hashes[key] = result_dict["Hash"]
