def content_find(self, content: Dict[str, Any]) -> Dict[str, Any]:
cnt_ids_bytes = {
algo_hash: hash_to_bytes(content[algo_hash])
for algo_hash in ALGORITHMS if content.get(algo_hash)
}
cnt = self.storage.content_find(cnt_ids_bytes)
return converters.from_content(cnt[0].to_dict()) if cnt else cnt
def lookup_content_raw(q: str) -> Dict[str, Any]:
"""Lookup the content defined by q.
Args:
q: query string of the form <hash_algo:hash>
Returns:
dict with 'sha1' and 'data' keys.
data representing its raw data decoded.
Raises:
NotFoundExc if the requested content is not found or
if the content bytes are not available in the storage
"""
c = lookup_content(q)
content_sha1_bytes = hashutil.hash_to_bytes(c["checksums"]["sha1"])
content_data = storage.content_get_data(content_sha1_bytes)
if content_data is None:
algo, hash_ = query.parse_hash(q)
raise NotFoundExc(
f"Bytes of content with {algo} checksum equals "
f"to {hashutil.hash_to_hex(hash_)} are not available!")
return converters.from_content({
"sha1": content_sha1_bytes,
"data": content_data
})
def content_get_data(self, cnt_id: str) -> Optional[Dict[str, Any]]:
cnt_id_bytes = hash_to_bytes(cnt_id)
cnt_data = self.storage.content_get_data(cnt_id_bytes)
if cnt_data is None:
return None
return converters.from_content({
"data": cnt_data,
"sha1": cnt_id_bytes
})
def lookup_hash(q):
"""Checks if the storage contains a given content checksum
Args: query string of the form <hash_algo:hash>
Returns: Dict with key found containing the hash info if the
hash is present, None if not.
"""
algo, hash = query.parse_hash(q)
found = storage.content_find({algo: hash})
return {'found': converters.from_content(found),
'algo': algo}
def lookup_content(q):
"""Lookup the content designed by q.
Args:
q: The release's sha1 as hexadecimal
Raises:
NotFoundExc if the requested content is not found
"""
algo, hash = query.parse_hash(q)
c = storage.content_find({algo: hash})
if not c:
raise NotFoundExc('Content with %s checksum equals to %s not found!' %
(algo, hashutil.hash_to_hex(hash)))
return converters.from_content(c)
def lookup_content(q: str) -> Dict[str, Any]:
"""Lookup the content designed by q.
Args:
q: The release's sha1 as hexadecimal
Raises:
NotFoundExc if the requested content is not found
"""
algo, hash_ = query.parse_hash(q)
c = _first_element(storage.content_find({algo: hash_}))
if not c:
hhex = hashutil.hash_to_hex(hash_)
raise NotFoundExc(
f"Content with {algo} checksum equals to {hhex} not found!")
return converters.from_content(c.to_dict())
def lookup_hash(q: str) -> Dict[str, Any]:
"""Check if the storage contains a given content checksum and return it if found.
Args:
q: query string of the form <hash_algo:hash>
Returns:
Dict with key found containing the hash info if the
hash is present, None if not.
"""
algo, hash_ = query.parse_hash(q)
found = _first_element(storage.content_find({algo: hash_}))
if found:
content = converters.from_content(found.to_dict())
else:
content = None
return {"found": content, "algo": algo}
def test_from_content(self):
content_input = {
'sha1':
hashutil.hash_to_bytes('5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5'),
'sha256':
hashutil.hash_to_bytes('39007420ca5de7cb3cfc15196335507e'
'e76c98930e7e0afa4d2747d3bf96c926'),
'blake2s256':
hashutil.hash_to_bytes('49007420ca5de7cb3cfc15196335507e'
'e76c98930e7e0afa4d2747d3bf96c926'),
'sha1_git':
hashutil.hash_to_bytes('40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03'),
'ctime':
'something-which-is-filtered-out',
'data':
b'data in bytes',
'length':
10,
'status':
'hidden',
}
# 'status' is filtered
expected_content = {
'checksums': {
'sha1': '5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5',
'sha256': '39007420ca5de7cb3cfc15196335507ee76c98'
'930e7e0afa4d2747d3bf96c926',
'blake2s256': '49007420ca5de7cb3cfc15196335507ee7'
'6c98930e7e0afa4d2747d3bf96c926',
'sha1_git': '40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03',
},
'data': b'data in bytes',
'length': 10,
'status': 'absent',
}
# when
actual_content = converters.from_content(content_input)
# then
self.assertEqual(actual_content, expected_content)
def test_from_content():
content_input = {
"sha1":
hashutil.hash_to_bytes("5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5"),
"sha256":
hashutil.hash_to_bytes("39007420ca5de7cb3cfc15196335507e"
"e76c98930e7e0afa4d2747d3bf96c926"),
"blake2s256":
hashutil.hash_to_bytes("49007420ca5de7cb3cfc15196335507e"
"e76c98930e7e0afa4d2747d3bf96c926"),
"sha1_git":
hashutil.hash_to_bytes("40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03"),
"ctime":
"something-which-is-filtered-out",
"data":
b"data in bytes",
"length":
10,
"status":
"hidden",
}
# 'status' is filtered
expected_content = {
"checksums": {
"sha1": "5c6f0e2750f48fa0bd0c4cf5976ba0b9e02ebda5",
"sha256": "39007420ca5de7cb3cfc15196335507ee76c98"
"930e7e0afa4d2747d3bf96c926",
"blake2s256": "49007420ca5de7cb3cfc15196335507ee7"
"6c98930e7e0afa4d2747d3bf96c926",
"sha1_git": "40e71b8614fcd89ccd17ca2b1d9e66c5b00a6d03",
},
"data": b"data in bytes",
"length": 10,
"status": "absent",
}
actual_content = converters.from_content(content_input)
assert actual_content == expected_content
def lookup_content_raw(q):
"""Lookup the content defined by q.
Args:
q: query string of the form <hash_algo:hash>
Returns:
dict with 'sha1' and 'data' keys.
data representing its raw data decoded.
Raises:
NotFoundExc if the requested content is not found or
if the content bytes are not available in the storage
"""
c = lookup_content(q)
content_sha1_bytes = hashutil.hash_to_bytes(c['checksums']['sha1'])
content = _first_element(storage.content_get([content_sha1_bytes]))
if not content:
algo, hash = query.parse_hash(q)
raise NotFoundExc('Bytes of content with %s checksum equals to %s '
'are not available!' %
(algo, hashutil.hash_to_hex(hash)))
return converters.from_content(content)
def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False):
"""Return information on directory pointed by revision with sha1_git.
If dir_path is not provided, display top level directory.
Otherwise, display the directory pointed by dir_path (if it exists).
Args:
sha1_git: revision's hash.
dir_path: optional directory pointed to by that revision.
with_data: boolean that indicates to retrieve the raw data if the path
resolves to a content. Default to False (for the api)
Returns:
Information on the directory pointed to by that revision.
Raises:
BadInputExc in case of unknown algo_hash or bad hash.
NotFoundExc either if the revision is not found or the path referenced
does not exist.
NotImplementedError in case of dir_path exists but do not reference a
type 'dir' or 'file'.
"""
sha1_git_bin = _to_sha1_bin(sha1_git)
revision = storage.revision_get([sha1_git_bin])[0]
if not revision:
raise NotFoundExc(f"Revision {sha1_git} not found")
dir_sha1_git_bin = revision.directory
if dir_path:
paths = dir_path.strip(os.path.sep).split(os.path.sep)
entity = storage.directory_entry_get_by_path(
dir_sha1_git_bin, list(map(lambda p: p.encode("utf-8"), paths)))
if not entity:
raise NotFoundExc(
"Directory or File '%s' pointed to by revision %s not found" %
(dir_path, sha1_git))
else:
entity = {"type": "dir", "target": dir_sha1_git_bin}
if entity["type"] == "dir":
directory_entries = storage.directory_ls(entity["target"]) or []
return {
"type":
"dir",
"path":
"." if not dir_path else dir_path,
"revision":
sha1_git,
"content":
list(map(converters.from_directory_entry, directory_entries)),
}
elif entity["type"] == "file": # content
content = _first_element(
storage.content_find({"sha1_git": entity["target"]}))
if not content:
raise NotFoundExc(f"Content not found for revision {sha1_git}")
content_d = content.to_dict()
if with_data:
data = storage.content_get_data(content.sha1)
if data:
content_d["data"] = data
return {
"type": "file",
"path": "." if not dir_path else dir_path,
"revision": sha1_git,
"content": converters.from_content(content_d),
}
elif entity["type"] == "rev": # revision
revision = storage.revision_get([entity["target"]])[0]
return {
"type": "rev",
"path": "." if not dir_path else dir_path,
"revision": sha1_git,
"content":
converters.from_revision(revision) if revision else None,
}
else:
raise NotImplementedError("Entity of type %s not implemented." %
entity["type"])
def test_from_content_none(self):
self.assertIsNone(converters.from_content(None))
def test_from_content_none():
assert converters.from_content(None) is None
def lookup_directory_with_revision(sha1_git, dir_path=None, with_data=False):
"""Return information on directory pointed by revision with sha1_git.
If dir_path is not provided, display top level directory.
Otherwise, display the directory pointed by dir_path (if it exists).
Args:
sha1_git: revision's hash.
dir_path: optional directory pointed to by that revision.
with_data: boolean that indicates to retrieve the raw data if the path
resolves to a content. Default to False (for the api)
Returns:
Information on the directory pointed to by that revision.
Raises:
BadInputExc in case of unknown algo_hash or bad hash.
NotFoundExc either if the revision is not found or the path referenced
does not exist.
NotImplementedError in case of dir_path exists but do not reference a
type 'dir' or 'file'.
"""
sha1_git_bin = _to_sha1_bin(sha1_git)
revision = _first_element(storage.revision_get([sha1_git_bin]))
if not revision:
raise NotFoundExc('Revision %s not found' % sha1_git)
dir_sha1_git_bin = revision['directory']
if dir_path:
paths = dir_path.strip(os.path.sep).split(os.path.sep)
entity = storage.directory_entry_get_by_path(
dir_sha1_git_bin, list(map(lambda p: p.encode('utf-8'), paths)))
if not entity:
raise NotFoundExc(
"Directory or File '%s' pointed to by revision %s not found"
% (dir_path, sha1_git))
else:
entity = {'type': 'dir', 'target': dir_sha1_git_bin}
if entity['type'] == 'dir':
directory_entries = storage.directory_ls(entity['target']) or []
return {'type': 'dir',
'path': '.' if not dir_path else dir_path,
'revision': sha1_git,
'content': list(map(converters.from_directory_entry,
directory_entries))}
elif entity['type'] == 'file': # content
content = storage.content_find({'sha1_git': entity['target']})
if with_data:
c = _first_element(storage.content_get([content['sha1']]))
content['data'] = c['data']
return {'type': 'file',
'path': '.' if not dir_path else dir_path,
'revision': sha1_git,
'content': converters.from_content(content)}
elif entity['type'] == 'rev': # revision
revision = next(storage.revision_get([entity['target']]))
return {'type': 'rev',
'path': '.' if not dir_path else dir_path,
'revision': sha1_git,
'content': converters.from_revision(revision)}
else:
raise NotImplementedError('Entity of type %s not implemented.'
% entity['type'])
def content_get(cls, cnt_id):
cnt_id_bytes = hash_to_bytes(cnt_id)
cnt = next(cls.storage.content_get([cnt_id_bytes]))
return converters.from_content(cnt)