def loadBulk(self, oids, returns=True):
"""
Load multiple objects at once
:param list oids: Iterable of oids to load
:param bool returns: When False, we don't return objects but store them
in cache
:return: List of (object, serial) tuples
:rtype: list
"""
if self.debug:
logging.debug("Loading: " + ", ".join([encode_hex(oid) for oid in oids]))
in_cache_before = {oid: oid in self._cache.current for oid in oids}
base_result = self.base.loadBulk(oids)
if self.debug:
in_cache_after = {oid: oid in self._cache.current for oid in oids}
if returns or self.debug:
datas, serials = zip(*base_result)
datas_out = map(self._untransform, datas)
out = list(zip(datas_out, serials))
if self.debug:
if datas:
self._debug_download_count += 1
for data, out_data, oid in zip(datas, datas_out, oids):
logline_prefix = ""
if not in_cache_before[oid]:
self._debug_download_size += len(data)
if not in_cache_after[oid]:
self._debug_download_count += 1
else:
logline_prefix = "(from bulk) "
logging.debug("%sid:%s, type:%s, transform: %s->%s" %
(logline_prefix, encode_hex(oid), debug_loads(out_data), len(data), len(out_data)))
if returns:
return out
def loadBefore(self, oid, tid):
"""Load last state for a given oid before a given tid
:param str oid: Object ID
:param str tid: Transaction timestamp
:return: Object and its serial number and following serial number
:rtype: tuple
"""
if self.debug:
if oid not in self._cache.current:
in_cache = False
else:
in_cache = True
data, serial, tend = self.base.loadBefore(oid, tid)
out_data = self._untransform(data)
if self.debug and not in_cache:
logging.debug(
"id:%s, type:%s, transform: %s->%s" % (
encode_hex(oid),
debug_loads(out_data),
len(data),
len(out_data),
))
self._debug_download_size += len(data)
self._debug_download_count += 1
return out_data, serial, tend
def loadBefore(self, oid, tid):
"""Load last state for a given oid before a given tid
:param str oid: Object ID
:param str tid: Transaction timestamp
:return: Object and its serial number and following serial number
:rtype: tuple
"""
if self.debug:
if oid not in self._cache.current:
in_cache = False
else:
in_cache = True
data, serial, tend = self.base.loadBefore(oid, tid)
out_data = self._untransform(data)
if self.debug and not in_cache:
logging.debug("id:%s, type:%s, transform: %s->%s" % (
encode_hex(oid),
debug_loads(out_data),
len(data),
len(out_data),
))
self._debug_download_size += len(data)
self._debug_download_count += 1
return out_data, serial, tend
def load(self, oid, version=''):
"""
Load object by oid
:param str oid: Object ID
:param version: Version to load (when we have version control)
:return: Object and its serial number
:rtype: tuple
"""
if self.debug:
if oid not in self._cache.current:
in_cache = False
else:
in_cache = True
data, serial = self.base.load(oid, version)
out_data = self._untransform(data)
if self.debug and not in_cache:
logging.debug("id:%s, type:%s, transform: %s->%s" %
(encode_hex(oid), debug_loads(out_data), len(data),
len(out_data)))
self._debug_download_size += len(data)
self._debug_download_count += 1
return out_data, serial
def loadBulk(self, oids, returns=True):
"""
Load multiple objects at once
:param list oids: Iterable of oids to load
:param bool returns: When False, we don't return objects but store them
in cache
:return: List of (object, serial) tuples
:rtype: list
"""
if self.debug:
logging.debug("Loading: " +
", ".join([encode_hex(oid) for oid in oids]))
in_cache_before = {oid: oid in self._cache.current for oid in oids}
base_result = self.base.loadBulk(oids)
if self.debug:
in_cache_after = {oid: oid in self._cache.current for oid in oids}
if returns or self.debug:
datas, serials = zip(*base_result)
datas_out = map(self._untransform, datas)
out = list(zip(datas_out, serials))
if self.debug:
if datas:
self._debug_download_count += 1
for data, out_data, oid in zip(datas, datas_out, oids):
logline_prefix = ""
if not in_cache_before[oid]:
self._debug_download_size += len(data)
if not in_cache_after[oid]:
self._debug_download_count += 1
else:
logline_prefix = "(from bulk) "
logging.debug(
"%sid:%s, type:%s, transform: %s->%s" %
(logline_prefix, encode_hex(oid),
debug_loads(out_data), len(data), len(out_data)))
if returns:
return out
def load(self, oid, version=''):
"""
Load object by oid
:param str oid: Object ID
:param version: Version to load (when we have version control)
:return: Object and its serial number
:rtype: tuple
"""
if self.debug:
if oid not in self._cache.current:
in_cache = False
else:
in_cache = True
data, serial = self.base.load(oid, version)
out_data = self._untransform(data)
if self.debug and not in_cache:
logging.debug("id:%s, type:%s, transform: %s->%s" % (encode_hex(oid), debug_loads(out_data), len(data), len(out_data)))
self._debug_download_size += len(data)
self._debug_download_count += 1
return out_data, serial
import zerodb from zerodb.crypto import ecc from zerodb.permissions import elliptic from zerodb.permissions import base as permissions_base from zerodb.storage import ZEOServer from zerodb.util import encode_hex from db import TEST_PASSPHRASE from db import create_objects_and_close, add_wiki_and_close TEST_PUBKEY = ecc.private(TEST_PASSPHRASE).get_pubkey() TEST_PUBKEY_3 = ecc.private(TEST_PASSPHRASE + " third").get_pubkey() TEST_PERMISSIONS = """realm ZERO root:%s third:%s""" % (encode_hex(TEST_PUBKEY), encode_hex(TEST_PUBKEY_3)) ZEO_CONFIG = """<zeo> address %(sock)s authentication-protocol ecc_auth authentication-database %(pass_file)s authentication-realm ZERO </zeo> <filestorage> path %(dbfile)s pack-gc false </filestorage>""" elliptic.register_auth()
def _check_permissions(self, data, oid=None):
if not data.endswith(self.user_id):
raise StorageError("Attempt to access encrypted data of others at <%s> by <%s>" % (oid, encode_hex(self.user_id)))
from zerodb.permissions import elliptic
from zerodb.permissions.base import PermissionsDatabase
from zerodb.storage import ZEOServer
from zerodb.util import encode_hex
kdf = elliptic.Client.kdf
TEST_PASSPHRASE = "v3ry 53cr3t pa$$w0rd"
TEST_PUBKEY = ecc.private(
TEST_PASSPHRASE, ("root", "ZERO"), kdf=kdf).get_pubkey()
TEST_PUBKEY_3 = ecc.private(
TEST_PASSPHRASE + " third", ("third", "ZERO"), kdf=kdf).get_pubkey()
TEST_PERMISSIONS = """realm ZERO
auth_secp256k1_scrypt:root:%s
auth_secp256k1_scrypt:third:%s""" % (encode_hex(TEST_PUBKEY), encode_hex(TEST_PUBKEY_3))
ZEO_CONFIG = """<zeo>
address %(sock)s
authentication-protocol auth_secp256k1_scrypt
authentication-database %(pass_file)s
authentication-realm ZERO
</zeo>
<filestorage>
path %(dbfile)s
pack-gc false
</filestorage>"""
elliptic.register_auth()
import zerodb
from zerodb.crypto import ecc
from zerodb.permissions import elliptic
from zerodb.permissions import base as permissions_base
from zerodb.storage import ZEOServer
from zerodb.util import encode_hex
from db import TEST_PASSPHRASE
from db import create_objects_and_close, add_wiki_and_close
TEST_PUBKEY = ecc.private(TEST_PASSPHRASE).get_pubkey()
TEST_PUBKEY_3 = ecc.private(TEST_PASSPHRASE + " third").get_pubkey()
TEST_PERMISSIONS = """realm ZERO
root:%s
third:%s""" % (
encode_hex(TEST_PUBKEY),
encode_hex(TEST_PUBKEY_3),
)
ZEO_CONFIG = """<zeo>
address %(sock)s
authentication-protocol ecc_auth
authentication-database %(pass_file)s
authentication-realm ZERO
</zeo>
<filestorage>
path %(dbfile)s
pack-gc false
</filestorage>"""