def api_directory(request, sha1_git, path=None):
"""
.. http:get:: /api/1/directory/(sha1_git)/[(path)/]
Get information about directory objects.
Directories are identified by **sha1** checksums, compatible with Git directory identifiers.
See :func:`swh.model.identifiers.directory_identifier` in our data model module for details
about how they are computed.
When given only a directory identifier, this endpoint returns information about the directory itself,
returning its content (usually a list of directory entries). When given a directory identifier and a
path, this endpoint returns information about the directory entry pointed by the relative path,
starting path resolution from the given directory.
:param string sha1_git: hexadecimal representation of the directory **sha1_git** identifier
:param string path: optional parameter to get information about the directory entry
pointed by that relative path
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept` header of request
:>jsonarr object checksums: object holding the computed checksum values for a directory entry
(only for file entries)
:>jsonarr string dir_id: **sha1_git** identifier of the requested directory
:>jsonarr number length: length of a directory entry in bytes (only for file entries)
for getting information about the content MIME type
:>jsonarr string name: the directory entry name
:>jsonarr number perms: permissions for the directory entry
:>jsonarr string target: **sha1_git** identifier of the directory entry
:>jsonarr string target_url: link to :http:get:`/api/1/content/[(hash_type):](hash)/`
or :http:get:`/api/1/directory/(sha1_git)/[(path)/]` depending on the directory entry type
:>jsonarr string type: the type of the directory entry, can be either ``dir``, ``file`` or ``rev``
**Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options`
:statuscode 200: no error
:statuscode 400: an invalid **hash_type** or **hash** has been provided
:statuscode 404: requested directory can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`directory/977fc4b98c0e85816348cebd3b12026407c368b6/`
""" # noqa
if path:
error_msg_path = ('Entry with path %s relative to directory '
'with sha1_git %s not found.') % (path, sha1_git)
return api_lookup(service.lookup_directory_with_path,
sha1_git,
path,
notfound_msg=error_msg_path,
enrich_fn=utils.enrich_directory)
else:
error_msg_nopath = 'Directory with sha1_git %s not found.' % sha1_git
return api_lookup(service.lookup_directory,
sha1_git,
notfound_msg=error_msg_nopath,
enrich_fn=utils.enrich_directory)
def test_api_lookup_not_found(self):
# when
with self.assertRaises(NotFoundExc) as e:
utils.api_lookup(
lambda x: None,
'something',
notfound_msg='this is the error message raised as it is None')
self.assertEqual(e.exception.args[0],
'this is the error message raised as it is None')
def test_genericapi_lookup_nothing_is_found():
def test_generic_lookup_fn(sha1, another_unused_arg):
assert another_unused_arg == "unused_arg"
assert sha1 == "sha1"
return None
notfound_msg = "This will be raised because None is returned."
with pytest.raises(NotFoundExc) as e:
utils.api_lookup(
test_generic_lookup_fn, "sha1", "unused_arg", notfound_msg=notfound_msg
)
assert e.match(notfound_msg)
def api_origin_visit_latest(request, origin_url=None):
"""
.. http:get:: /api/1/origin/(origin_url)/visit/latest/
Get information about the latest visit of a software origin.
:param str origin_url: a software origin URL
:query boolean require_snapshot: if true, only return a visit
with a snapshot
{common_headers}
{return_origin_visit}
:statuscode 200: no error
:statuscode 404: requested origin or visit can not be found in the
archive
**Example:**
.. parsed-literal::
:swh_web_api:`origin/https://github.com/hylang/hy/visit/latest/`
"""
require_snapshot = request.query_params.get("require_snapshot", "false")
return api_lookup(
archive.lookup_origin_visit_latest,
origin_url,
bool(strtobool(require_snapshot)),
notfound_msg=("No visit for origin {} found".format(origin_url)),
enrich_fn=partial(enrich_origin_visit,
with_origin_link=True,
with_origin_visit_link=False),
request=request,
)
def api_origin_visit(request, visit_id, origin_url):
"""
.. http:get:: /api/1/origin/(origin_url)/visit/(visit_id)/
Get information about a specific visit of a software origin.
:param str origin_url: a software origin URL
:param int visit_id: a visit identifier
{common_headers}
{return_origin_visit}
:statuscode 200: no error
:statuscode 404: requested origin or visit can not be found in the
archive
**Example:**
.. parsed-literal::
:swh_web_api:`origin/https://github.com/hylang/hy/visit/1/`
"""
return api_lookup(
archive.lookup_origin_visit,
origin_url,
int(visit_id),
notfound_msg=("No visit {} for origin {} found".format(
visit_id, origin_url)),
enrich_fn=partial(enrich_origin_visit,
with_origin_link=True,
with_origin_visit_link=False),
request=request,
)
def api_origin_intrinsic_metadata(request, origin_url):
"""
.. http:get:: /api/1/origin/(origin_url)/intrinsic-metadata
Get intrinsic metadata of a software origin (as a JSON-LD/CodeMeta dictionary).
:param string origin_url: the origin url
:>json string ???: intrinsic metadata field of the origin
{common_headers}
:statuscode 200: no error
:statuscode 404: requested origin can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`origin/https://github.com/python/cpython/intrinsic-metadata`
"""
return api_lookup(
archive.lookup_origin_intrinsic_metadata,
origin_url,
notfound_msg=f"Origin with url {origin_url} not found",
enrich_fn=enrich_origin,
request=request,
)
def api_person(request, person_id):
"""
.. http:get:: /api/1/person/(person_id)/
Get information about a person in the archive.
:param int person_id: a person identifier
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept` header of request
:>json string email: the email of the person
:>json string fullname: the full name of the person: combination of its name and email
:>json number id: the unique identifier of the person
:>json string name: the name of the person
**Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options`
:statuscode 200: no error
:statuscode 404: requested person can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`person/8275/`
""" # noqa
return api_lookup(
service.lookup_person,
person_id,
notfound_msg='Person with id {} not found.'.format(person_id))
def test_api_lookup_with_result_as_map(self):
# when
actual_result = utils.api_lookup(
lambda x: map(lambda y: y + 1, x), [1, 2, 3],
notfound_msg='this is the error which won\'t be used here')
self.assertEqual(actual_result, [2, 3, 4])
def api_revision(request, sha1_git):
"""
.. http:get:: /api/1/revision/(sha1_git)/
Get information about a revision in the archive. Revisions are
identified by **sha1** checksums, compatible with Git commit
identifiers.
See :func:`swh.model.identifiers.revision_identifier` in our data model
module for details about how they are computed.
:param string sha1_git: hexadecimal representation of the revision
**sha1_git** identifier
{common_headers}
{return_revision}
:statuscode 200: no error
:statuscode 400: an invalid **sha1_git** value has been provided
:statuscode 404: requested revision can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/`
"""
return api_lookup(
archive.lookup_revision,
sha1_git,
notfound_msg="Revision with sha1_git {} not found.".format(sha1_git),
enrich_fn=utils.enrich_revision,
request=request,
)
def api_vault_fetch_directory(request, dir_id):
"""
.. http:get:: /api/1/vault/directory/(dir_id)/raw/
Fetch the cooked archive for a directory.
See :http:get:`/api/1/vault/directory/(dir_id)/` to get more
details on directory cooking.
:param string dir_id: the directory's sha1 identifier
:resheader Content-Type: application/octet-stream
**Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options`
:statuscode 200: no error
:statuscode 400: an invalid directory identifier has been provided
:statuscode 404: requested directory can not be found in the archive
""" # noqa
_, obj_id = query.parse_hash_with_algorithms_or_throws(
dir_id, ['sha1'], 'Only sha1_git is supported.')
res = api_lookup(
service.vault_fetch,
'directory',
obj_id,
notfound_msg="Directory with ID '{}' not found.".format(dir_id))
fname = '{}.tar.gz'.format(dir_id)
response = HttpResponse(res, content_type='application/gzip')
response['Content-disposition'] = 'attachment; filename={}'.format(fname)
return response
def api_vault_fetch_revision_gitfast(request, rev_id):
"""
.. http:get:: /api/1/vault/revision/(rev_id)/gitfast/raw/
Fetch the cooked gitfast archive for a revision.
See :http:get:`/api/1/vault/revision/(rev_id)/gitfast/` to get more
details on directory cooking.
:param string rev_id: the revision's sha1 identifier
:resheader Content-Type: application/octet-stream
:statuscode 200: no error
:statuscode 400: an invalid revision identifier has been provided
:statuscode 404: requested directory did not receive any cooking
request yet (in case of GET) or can not be found in the archive
(in case of POST)
"""
_, obj_id = query.parse_hash_with_algorithms_or_throws(
rev_id, ["sha1"], "Only sha1_git is supported."
)
res = api_lookup(
archive.vault_fetch,
"revision_gitfast",
obj_id,
notfound_msg="Cooked archive for revision '{}' not found.".format(rev_id),
request=request,
)
fname = "{}.gitfast.gz".format(rev_id)
response = HttpResponse(res, content_type="application/gzip")
response["Content-disposition"] = "attachment; filename={}".format(fname)
return response
def api_origin(request, origin_url):
"""
.. http:get:: /api/1/origin/(origin_url)/get/
Get information about a software origin.
:param string origin_url: the origin url
{return_origin}
{common_headers}
:statuscode 200: no error
:statuscode 404: requested origin can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`origin/https://github.com/python/cpython/get/`
"""
ori_dict = {"url": origin_url}
error_msg = "Origin with url %s not found." % ori_dict["url"]
return api_lookup(
archive.lookup_origin,
ori_dict,
notfound_msg=error_msg,
enrich_fn=enrich_origin,
request=request,
)
def test_genericapi_lookup_nothing_is_found(self):
# given
def test_generic_lookup_fn(sha1, another_unused_arg):
assert another_unused_arg == 'unused_arg'
assert sha1 == 'sha1'
return None
# when
with self.assertRaises(NotFoundExc) as cm:
utils.api_lookup(
test_generic_lookup_fn,
'sha1',
'unused_arg',
notfound_msg='This will be raised because None is returned.')
self.assertIn('This will be raised because None is returned.',
cm.exception.args[0])
def test_api_lookup_with_result(self):
# when
actual_result = utils.api_lookup(
lambda x: x + '!',
'something',
notfound_msg='this is the error which won\'t be used here')
self.assertEqual(actual_result, 'something!')
def test_api_lookup_with_result_as_map():
actual_result = utils.api_lookup(
lambda x: map(lambda y: y + 1, x),
[1, 2, 3],
notfound_msg="this is the error which won't be used here",
)
assert actual_result == [2, 3, 4]
def test_api_lookup_with_result():
actual_result = utils.api_lookup(
lambda x: x + "!",
"something",
notfound_msg="this is the error which won't be used here",
)
assert actual_result == "something!"
def _dispatch_cook_progress(request, obj_type, obj_id):
hex_id = hashutil.hash_to_hex(obj_id)
object_name = obj_type.split('_')[0].title()
if request.method == 'GET':
return api_lookup(service.vault_progress,
obj_type,
obj_id,
notfound_msg=("{} '{}' was never requested.".format(
object_name, hex_id)))
elif request.method == 'POST':
email = request.POST.get('email', request.GET.get('email', None))
return api_lookup(service.vault_cook,
obj_type,
obj_id,
email,
notfound_msg=("{} '{}' not found.".format(
object_name, hex_id)))
def api_content_ctags(request, q):
"""
Get information about all `Ctags <http://ctags.sourceforge.net/>`_-style
symbols defined in a content object.
"""
return api_lookup(
service.lookup_content_ctags,
q,
notfound_msg='No ctags symbol found for content {}.'.format(q),
enrich_fn=utils.enrich_metadata_endpoint)
def api_origins(request):
"""
.. http:get:: /api/1/origins/
Get list of archived software origins.
Origins are sorted by ids before returning them.
:query int origin_from: The first origin id that will be included
in returned results (default to 1)
:query int origin_count: The maximum number of origins to return
(default to 100, can not exceed 10000)
:>jsonarr number id: the origin unique identifier
:>jsonarr string origin_visits_url: link to in order to get information about the
visits for that origin
:>jsonarr string type: the type of software origin (possible values are ``git``, ``svn``,
``hg``, ``deb``, ``pypi``, ``ftp`` or ``deposit``)
:>jsonarr string url: the origin canonical url
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept` header of request
:resheader Link: indicates that a subsequent or previous result page are available
and contains the urls pointing to them
**Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options`
:statuscode 200: no error
**Example:**
.. parsed-literal::
:swh_web_api:`origins?origin_from=50000&origin_count=500`
""" # noqa
origin_from = int(request.query_params.get('origin_from', '1'))
origin_count = int(request.query_params.get('origin_count', '100'))
origin_count = min(origin_count, 10000)
results = api_lookup(service.lookup_origins,
origin_from,
origin_count + 1,
enrich_fn=_enrich_origin)
response = {'results': results, 'headers': {}}
if len(results) > origin_count:
origin_from = results.pop()['id']
response['headers']['link-next'] = reverse('api-origins',
query_params={
'origin_from':
origin_from,
'origin_count':
origin_count
})
return response
def api_origin_visit(request, origin_id, visit_id):
"""
.. http:get:: /api/1/origin/(origin_id)/visit/(visit_id)/
Get information about a specific visit of a software origin.
:param int origin_id: a software origin identifier
:param int visit_id: a visit identifier
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept` header of request
:>json string date: ISO representation of the visit date (in UTC)
:>json number origin: the origin unique identifier
:>json string origin_url: link to get information about the origin
:>jsonarr string snapshot: the snapshot identifier of the visit
:>jsonarr string snapshot_url: link to :http:get:`/api/1/snapshot/(snapshot_id)/`
in order to get information about the snapshot of the visit
:>json string status: status of the visit (either **full**, **partial** or **ongoing**)
:>json number visit: the unique identifier of the visit
**Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options`
:statuscode 200: no error
:statuscode 404: requested origin or visit can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`origin/1500/visit/1/`
""" # noqa
def _enrich_origin_visit(origin_visit):
ov = origin_visit.copy()
ov['origin_url'] = reverse('api-origin',
url_args={'origin_id': ov['origin']})
snapshot = ov['snapshot']
if snapshot:
ov['snapshot_url'] = reverse('api-snapshot',
url_args={'snapshot_id': snapshot})
else:
ov['snapshot_url'] = None
return ov
return api_lookup(service.lookup_origin_visit,
int(origin_id),
int(visit_id),
notfound_msg=('No visit {} for origin {} found'.format(
visit_id, origin_id)),
enrich_fn=_enrich_origin_visit)
def api_revision_log(request, sha1_git):
"""
.. http:get:: /api/1/revision/(sha1_git)/log/
Get a list of all revisions heading to a given one, in other words show
the commit log.
The revisions are returned in the breadth-first search order while
visiting the revision graph. The number of revisions to return is also
bounded by the **limit** query parameter.
.. warning::
To get the full BFS traversal of the revision graph when the
total number of revisions is greater than 1000, it is up to
the client to keep track of the multiple branches of history
when there's merge revisions in the returned objects.
In other words, identify all the continuation points that need
to be followed to get the full history through recursion.
:param string sha1_git: hexadecimal representation of the revision
**sha1_git** identifier
:query int limit: maximum number of revisions to return when performing
BFS traversal on the revision graph (default to 10, can not exceed 1000)
{common_headers}
{return_revision_array}
:statuscode 200: no error
:statuscode 400: an invalid **sha1_git** value has been provided
:statuscode 404: head revision can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`revision/e1a315fa3fa734e2a6154ed7b5b9ae0eb8987aad/log/`
"""
limit = int(request.query_params.get("limit", "10"))
limit = min(limit, 1000)
error_msg = "Revision with sha1_git %s not found." % sha1_git
revisions = api_lookup(
archive.lookup_revision_log,
sha1_git,
limit,
notfound_msg=error_msg,
enrich_fn=utils.enrich_revision,
request=request,
)
return {"results": revisions}
def api_revision(request, sha1_git):
"""
.. http:get:: /api/1/revision/(sha1_git)/
Get information about a revision in the archive.
Revisions are identified by **sha1** checksums, compatible with Git commit identifiers.
See :func:`swh.model.identifiers.revision_identifier` in our data model module for details
about how they are computed.
:param string sha1_git: hexadecimal representation of the revision **sha1_git** identifier
:reqheader Accept: the requested response content type,
either ``application/json`` (default) or ``application/yaml``
:resheader Content-Type: this depends on :http:header:`Accept` header of request
:>json object author: information about the author of the revision
:>json string author_url: link to :http:get:`/api/1/person/(person_id)/` to get
information about the author of the revision
:>json object committer: information about the committer of the revision
:>json string committer_url: link to :http:get:`/api/1/person/(person_id)/` to get
information about the committer of the revision
:>json string committer_date: ISO representation of the commit date (in UTC)
:>json string date: ISO representation of the revision date (in UTC)
:>json string directory: the unique identifier that revision points to
:>json string directory_url: link to :http:get:`/api/1/directory/(sha1_git)/[(path)/]`
to get information about the directory associated to the revision
:>json string id: the revision unique identifier
:>json boolean merge: whether or not the revision corresponds to a merge commit
:>json string message: the message associated to the revision
:>json array parents: the parents of the revision, i.e. the previous revisions
that head directly to it, each entry of that array contains an unique parent
revision identifier but also a link to :http:get:`/api/1/revision/(sha1_git)/`
to get more information about it
:>json string type: the type of the revision
**Allowed HTTP Methods:** :http:method:`get`, :http:method:`head`, :http:method:`options`
:statuscode 200: no error
:statuscode 400: an invalid **sha1_git** value has been provided
:statuscode 404: requested revision can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`revision/aafb16d69fd30ff58afdd69036a26047f3aebdc6/`
""" # noqa
return api_lookup(
service.lookup_revision,
sha1_git,
notfound_msg='Revision with sha1_git {} not found.'.format(sha1_git),
enrich_fn=utils.enrich_revision)
def test_generic_api_generator_are_enriched_and_returned_as_list():
def test_generic_lookup_fn_3(crit):
assert crit == "crit"
return (i for i in [4, 5, 6])
actual_result = utils.api_lookup(
test_generic_lookup_fn_3,
"crit",
notfound_msg="Move!",
enrich_fn=lambda x, request: x - 1,
)
assert actual_result == [3, 4, 5]
def test_generic_api_generator_are_enriched_and_returned_as_list(self):
# given
def test_generic_lookup_fn_3(crit):
assert crit == 'crit'
return (i for i in [4, 5, 6])
# when
actual_result = utils.api_lookup(test_generic_lookup_fn_3,
'crit',
notfound_msg='Move!',
enrich_fn=lambda x: x - 1)
self.assertEqual(actual_result, [3, 4, 5])
def _dispatch_cook_progress(request, obj_type, obj_id):
hex_id = hashutil.hash_to_hex(obj_id)
object_name = obj_type.split("_")[0]
if request.method == "GET":
return api_lookup(
archive.vault_progress,
obj_type,
obj_id,
notfound_msg=(
"Cooking of {} '{}' was never requested.".format(object_name, hex_id)
),
request=request,
)
elif request.method == "POST":
email = request.POST.get("email", request.GET.get("email", None))
return api_lookup(
archive.vault_cook,
obj_type,
obj_id,
email,
notfound_msg=("{} '{}' not found.".format(object_name.title(), hex_id)),
request=request,
)
def api_content_metadata(request, q):
"""
.. http:get:: /api/1/content/[(hash_type):](hash)/
Get information about a content (aka a "blob") object.
In the archive, a content object is identified based on checksum
values computed using various hashing algorithms.
:param string hash_type: optional parameter specifying which hashing algorithm
has been used to compute the content checksum. It can be either ``sha1``,
``sha1_git``, ``sha256`` or ``blake2s256``. If that parameter is not
provided, it is assumed that the hashing algorithm used is ``sha1``.
:param string hash: hexadecimal representation of the checksum value computed
with the specified hashing algorithm.
{common_headers}
:>json object checksums: object holding the computed checksum values for the
requested content
:>json string data_url: link to
:http:get:`/api/1/content/[(hash_type):](hash)/raw/`
for downloading the content raw bytes
:>json string filetype_url: link to
:http:get:`/api/1/content/[(hash_type):](hash)/filetype/`
for getting information about the content MIME type
:>json string language_url: link to
:http:get:`/api/1/content/[(hash_type):](hash)/language/`
for getting information about the programming language used in the content
:>json number length: length of the content in bytes
:>json string license_url: link to
:http:get:`/api/1/content/[(hash_type):](hash)/license/`
for getting information about the license of the content
:statuscode 200: no error
:statuscode 400: an invalid **hash_type** or **hash** has been provided
:statuscode 404: requested content can not be found in the archive
**Example:**
.. parsed-literal::
:swh_web_api:`content/sha1_git:fe95a46679d128ff167b7c55df5d02356c5a1ae1/`
"""
return api_lookup(
archive.lookup_content,
q,
notfound_msg="Content with {} not found.".format(q),
enrich_fn=functools.partial(utils.enrich_content, query_string=q),
request=request,
)
def test_generic_api_list_are_enriched_too():
def test_generic_lookup_fn_2(crit):
assert crit == "something"
return ["a", "b", "c"]
actual_result = utils.api_lookup(
test_generic_lookup_fn_2,
"something",
notfound_msg=(
"Not the error message you are looking for, it is. " "Along, you move!"
),
enrich_fn=lambda x, request: "".join(["=", x, "="]),
)
assert actual_result == ["=a=", "=b=", "=c="]
def test_generic_api_list_are_enriched_too(self):
# given
def test_generic_lookup_fn_2(crit):
assert crit == 'something'
return ['a', 'b', 'c']
# when
actual_result = utils.api_lookup(
test_generic_lookup_fn_2,
'something',
notfound_msg=('Not the error message you are looking for, it is. '
'Along, you move!'),
enrich_fn=lambda x: ''.join(['=', x, '=']))
self.assertEqual(actual_result, ['=a=', '=b=', '=c='])
def test_generic_api_map_are_enriched_and_transformed_to_list(self):
# given
def test_generic_lookup_fn_1(criteria0, param0, param1):
assert criteria0 == 'something'
return map(lambda x: x + 1, [1, 2, 3])
# when
actual_result = utils.api_lookup(
test_generic_lookup_fn_1,
'something',
'some param 0',
'some param 1',
notfound_msg=('This is not the error message you are looking for. '
'Move along.'),
enrich_fn=lambda x: x * 2)
self.assertEqual(actual_result, [4, 6, 8])
def test_generic_api_simple_data_are_enriched_and_returned_too(self):
# given
def test_generic_lookup_fn_4(crit):
assert crit == '123'
return {'a': 10}
def test_enrich_data(x):
x['a'] = x['a'] * 10
return x
# when
actual_result = utils.api_lookup(test_generic_lookup_fn_4,
'123',
notfound_msg='Nothing to do',
enrich_fn=test_enrich_data)
self.assertEqual(actual_result, {'a': 100})
