def write_ipfs_uris_to_disk(ethpm_dir: Path,
manifests: Dict[Address, Dict[str, str]]) -> None:
all_manifest_uris = [
version_release_data["manifestURI"]
for version_release_data in manifests.values() if
is_supported_content_addressed_uri(version_release_data["manifestURI"])
]
base_ipfs_uris = [uri for uri in all_manifest_uris if is_ipfs_uri(uri)]
nested_ipfs_uris = [
pluck_ipfs_uris_from_manifest(uri) for uri in all_manifest_uris
]
all_ipfs_uris = set(flatten(nested_ipfs_uris) + base_ipfs_uris)
# ex.
# ipfs uri: QmdvZEW3AaUntDfFkcbdnYzeLAAeD4YFeixQsdmHF88T6Q
# dir store: ethpmcli/Qm/dv/ZE/QmdvZEW3AaUntDfFkcbdnYzeLAAeD4YFeixQsdmHF88T6Q
for uri in all_ipfs_uris:
ipfs_hash = extract_ipfs_path_from_uri(uri)
first_two_bytes_dir = ethpm_dir / ipfs_hash[0:2]
second_two_bytes_dir = first_two_bytes_dir / ipfs_hash[2:4]
third_two_bytes_dir = second_two_bytes_dir / ipfs_hash[4:6]
asset_dest_path = third_two_bytes_dir / ipfs_hash
if not asset_dest_path.is_file():
if not first_two_bytes_dir.is_dir():
first_two_bytes_dir.mkdir()
if not second_two_bytes_dir.is_dir():
second_two_bytes_dir.mkdir()
if not third_two_bytes_dir.is_dir():
third_two_bytes_dir.mkdir()
asset_dest_path.touch()
asset_dest_path.write_bytes(resolve_uri_contents(uri))
logger.info("%s written to\n %s.\n", uri, asset_dest_path)
def fetch_uri_contents(self, uri: str) -> bytes:
ipfs_hash = extract_ipfs_path_from_uri(uri)
contents = self.client.cat(ipfs_hash)
validation_hash = generate_file_hash(contents)
if validation_hash != ipfs_hash:
raise ValidationError(
f"Hashed IPFS contents retrieved from uri: {uri} do not match its content hash."
)
return contents
def fetch_uri_contents(self, uri: str) -> bytes:
ipfs_hash = extract_ipfs_path_from_uri(uri)
contents = self.client.cat(ipfs_hash)
# Local validation of hashed contents only works for non-chunked files ~< 256kb
# Improved validation WIP @ https://github.com/ethpm/py-ethpm/pull/165
if len(contents) <= 262144:
validation_hash = generate_file_hash(contents)
if validation_hash != ipfs_hash:
raise EthPMValidationError(
f"Hashed IPFS contents retrieved from uri: {uri} do not match its content hash."
)
return contents
def resolve_manifest_uri(uri: URI,
ipfs: BaseIPFSBackend) -> ResolvedManifestURI:
github_backend = GithubOverHTTPSBackend()
if github_backend.can_resolve_uri(uri):
raw_manifest = github_backend.fetch_uri_contents(uri)
resolved_content_hash = parse.urlparse(uri).path.split("/")[-1]
elif ipfs.can_resolve_uri(uri):
raw_manifest = ipfs.fetch_uri_contents(uri)
resolved_content_hash = extract_ipfs_path_from_uri(uri)
else:
raise UriNotSupportedError(
f"{uri} is not supported. Currently ethPM CLI only supports "
"IPFS and Github blob manifest uris.")
return ResolvedManifestURI(raw_manifest, resolved_content_hash)
def test_extract_ipfs_path_from_uri(value, expected):
actual = extract_ipfs_path_from_uri(value)
assert actual == expected
