def get_size_multi(self, *blobrefs):
"""
Get the size of several blobs at once, given their blobrefs.
This is a batch version of :py:meth:`get_size`, returning a
mapping object whose keys are the request blobrefs and whose
values are either the size of each corresponding blob or
``None`` if the blobref is not known to the server.
"""
import json
form_data = {}
form_data["camliversion"] = "1"
for i, blobref in enumerate(blobrefs):
form_data["blob%i" % (i + 1)] = blobref
stat_url = self._make_url('camli/stat')
resp = self.http_session.post(stat_url, data=form_data)
if resp.status_code != 200:
from camlistore.exceptions import ServerError
raise ServerError("Failed to get sizes of blobs: got %i %s" % (
resp.status_code,
resp.reason,
))
data = json.loads(resp.content)
ret = {blobref: None for blobref in blobrefs}
for raw_meta in data["stat"]:
ret[raw_meta["blobRef"]] = int(raw_meta["size"])
return ret
def get_claims_for_permanode(self, blobref):
"""
Get the claims for a particular permanode, as an iterable of
:py:class:`ClaimMeta`.
The concept of "claims" is what allows a permanode to appear
mutable even though the underlying storage is immutable. The
indexer processes each of the valid claims on a given permanode
to produce an aggregated set of its attributes for a given point
in time.
Most callers should prefer to use :py:meth:`describe_blob` instead,
since that returns the flattened result of processing all
attributes, rather than requiring the client to process the claims
itself.
"""
import json
req_url = self._make_url("camli/search/claims")
resp = self.http_session.get(
req_url,
params={"permanode": blobref},
)
if resp.status_code != 200:
from camlistore.exceptions import ServerError
raise ServerError(
"Failed to get claims for %s: server returned %i %s" % (
blobref,
resp.status_code,
resp.reason,
))
raw = json.loads(resp.content)
return [ClaimMeta(x) for x in raw["claims"]]
def put_multi(self, *blobs):
"""
Upload several blobs to the store.
This is a batch version of :py:meth:`put`, uploading several
blobs at once and returning a list of their blobrefs in the
same order as they were provided in the arguments.
At present this method does *not* correctly handle the protocol
restriction that only 32MB of data can be uploaded at once, so
this function will fail if that limit is exceeded. It is intended
that this will be fixed in a future version.
"""
import hashlib
upload_url = self._make_url('camli/upload')
blobrefs = [blob.blobref for blob in blobs]
sizes = self.get_size_multi(*blobrefs)
files_to_post = {}
for blob in blobs:
blobref = blob.blobref
if sizes[blobref] is not None:
# Server already has this blob, so skip
continue
files_to_post[blobref] = (
blobref,
blob.data,
'application/octet-stream',
)
if len(files_to_post) == 0:
# Server already has everything, so nothing to do.
return blobrefs
# FIXME: We should detect if our total upload size is >32MB
# and automatically split it into multiple requests, since the
# protocol forbids upload payloads greater than 32MB.
resp = self.http_session.post(upload_url, files=files_to_post)
if resp.status_code != 200:
from camlistore.exceptions import ServerError
raise ServerError("Failed to upload blobs: got %i %s" % (
resp.status_code,
resp.reason,
))
return blobrefs
def enumerate(self):
"""
Enumerate all of the blobs on the server, in blobref order.
Returns an iterable over all of the blobs. The underlying server
interface returns the resultset in chunks, so beginning iteration
will cause one request but continued iteration may cause followup
requests to retrieve additional chunks.
Most applications do not need to enumerate all blobs and can instead
use the facilities provided by the search interface. The enumeration
interface exists primarily to enable the Camlistore indexer to build
its search index, but may be useful for other alternative index
implementations.
"""
from urllib.parse import urljoin
import json
plain_enum_url = self._make_url("camli/enumerate-blobs")
next_enum_url = plain_enum_url
while next_enum_url is not None:
resp = self.http_session.get(next_enum_url)
if resp.status_code != 200:
from camlistore.exceptions import ServerError
raise ServerError(
"Failed to enumerate blobs from %s: got %i %s" % (
next_enum_url,
resp.status_code,
resp.reason,
))
data = json.loads(resp.content)
if "continueAfter" in data:
next_enum_url = urljoin(
plain_enum_url,
"?after=" + data["continueAfter"],
)
else:
next_enum_url = None
for raw_blob_reference in data["blobs"]:
yield BlobMeta(
raw_blob_reference["blobRef"],
size=raw_blob_reference["size"],
blob_client=self,
)
def query(self, expression):
"""
Run a query against the index, returning an iterable of
:py:class:`SearchResult`.
The given expression is just passed on verbatim to the underlying
query interface.
Query constraints are not yet supported.
"""
import json
req_url = self._make_url("camli/search/query")
data = {
# TODO: Understand how constraints work and implement them
# https://github.com/bradfitz/camlistore/blob/
# ca58231336e5711abacb059763beb06e8b2b1788/pkg/search/query.go#L255
# "constraint": "",
"expression": expression,
}
resp = self.http_session.post(
req_url,
data=json.dumps(data),
)
if resp.status_code != 200:
from camlistore.exceptions import ServerError
raise ServerError(
"Failed to search for %r: server returned %i %s" % (
expression,
resp.status_code,
resp.reason,
))
raw_data = json.loads(resp.content)
if raw_data["blobs"] is not None:
return [SearchResult(x["blob"]) for x in raw_data["blobs"]]
else:
return []
def get_size(self, blobref):
"""
Get the size of a blob, given its blobref.
Returns the size of the blob as an :py:class:`int` in bytes,
or raises :py:class:`camlistore.exceptions.NotFoundError` if
the given blobref is not known to the server.
"""
blob_url = self._make_blob_url(blobref)
resp = self.http_session.request('HEAD', blob_url)
if resp.status_code == 200:
return int(resp.headers['content-length'])
elif resp.status_code == 404:
from camlistore.exceptions import NotFoundError
raise NotFoundError("Blob not found: %s" % blobref, )
else:
from camlistore.exceptions import ServerError
raise ServerError(
"Failed to get metadata for blob %s: server returned %i %s" % (
blobref,
resp.status_code,
resp.reason,
))
def get(self, blobref):
"""
Get the data for a blob, given its blobref.
Returns a :py:class:`camlistore.Blob` instance describing the
blob, or raises :py:class:`camlistore.exceptions.NotFoundError` if
the given blobref is not known to the server.
"""
blob_url = self._make_blob_url(blobref)
resp = self.http_session.get(blob_url)
if resp.status_code == 200:
return Blob(resp.content, blobref=blobref)
elif resp.status_code == 404:
from camlistore.exceptions import NotFoundError
raise NotFoundError("Blob not found: %s" % blobref, )
else:
from camlistore.exceptions import ServerError
raise ServerError("Failed to get blob %s: server returned %i %s" %
(
blobref,
resp.status_code,
resp.reason,
))
def describe_blob(self, blobref):
"""
Request a description of a particular blob, returning a
:py:class:`BlobDescription` object.
The "description" of a blob is the indexer's record of the blob,
so it contains only the subset of information retained by the
indexer. The level of detail in the returned object will thus
depend on what the indexer knows about the given object.
"""
import json
req_url = self._make_url("camli/search/describe")
resp = self.http_session.get(
req_url,
params={
"blobref": blobref,
},
)
if resp.status_code != 200:
from camlistore.exceptions import ServerError
raise ServerError("Failed to describe %s: server returned %i %s" %
(
blobref,
resp.status_code,
resp.reason,
))
raw = json.loads(resp.content)
my_raw = raw["meta"][blobref]
other_raw = raw["meta"]
return BlobDescription(
self,
my_raw,
other_raw_dicts=other_raw,
)