def _init_db(self):
"""We need this to be executed each time we are in a new process"""
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = DB.db_factory(self._storage, **self.__db_kwargs)
self._conn_open()
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
if self._autoreindex:
subscribers.init()
Random.atfork()
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = SubDB(self._storage, **self.__db_kwargs)
self._conn_open()
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = DB.db_factory(self._storage, **self.__db_kwargs)
self._conn_open()
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
if self._autoreindex:
subscribers.init()
Random.atfork()
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = SubDB(self._storage, **self.__db_kwargs)
self._conn_open()
class DB(object):
"""
Database for this user. Everything is used through this class
"""
db_factory = subdb.DB
auth_module = elliptic
encrypter = AES256Encrypter
compressor = None
def __init__(self, sock, username=None, password=None, realm="ZERO", debug=False, pool_timeout=3600, pool_size=7, **kw):
"""
:param str sock: UNIX (str) or TCP ((str, int)) socket
:type sock: str or tuple
:param str username: Username. Derived from password if not set
:param str password: Password or seed for private key
:param str realm: ZODB's realm
:param bool debug: Whether to log debug messages
"""
# ZODB doesn't like unicode here
username = username and str(username)
password = str(password)
if isinstance(sock, basestring):
sock = str(sock)
elif type(sock) in (list, tuple):
assert len(sock) == 2
sock = str(sock[0]), int(sock[1])
self.auth_module = kw.pop("auth_module", self.auth_module)
self.auth_module.register_auth()
if not username:
username = sha256("username" + sha256(password).digest()).digest()
self._init_default_crypto(passphrase=password)
# Store all the arguments necessary for login in this instance
self.__storage_kwargs = {
"sock": sock,
"username": username,
"password": password,
"realm": realm,
"debug": debug}
self.__db_kwargs = {
"pool_size": pool_size,
"pool_timeout": pool_timeout,
"cache_size": 50000,
"cache_size_bytes": 2 ** 30}
self.__db_kwargs.update(kw)
# For multi-threading
self.__pid = os.getpid()
self._init_db()
self._models = {}
def _init_default_crypto(self, passphrase=None):
if self.encrypter:
self.encrypter.register_class(default=True)
if self.compressor:
self.compressor.register(default=True)
init_crypto(passphrase=passphrase)
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = DB.db_factory(self._storage, **self.__db_kwargs)
self._conn_open()
def _conn_open(self):
"""Opens db connection and registers a destuction callback"""
self.__thread_local.conn = conn = self._db.open()
def destructor(conn_id):
conn = self._db.pool.all.data.get(conn_id, None)
if conn:
# Hack to make it not closing TM which is already removed
conn.opened = 0
conn.close()
self.__thread_watcher.watch(destructor, id(conn))
@property
def _root(self):
"""Access database root for this user"""
if os.getpid() != self.__pid:
# If a new process spins up, we need to re-initialize everything
self.__pid = os.getpid()
self._init_db()
else:
# Open connections from the pool when new threads spin up
# Should be closed when old thread-locals get garbage collected
if not hasattr(self.__thread_local, "conn") or\
self.__thread_local.conn.opened is None:
self._conn_open()
return self.__thread_local.conn.root()
@property
def _connection(self):
return self.__thread_local.conn
def disconnect(self):
if hasattr(self.__thread_local, "conn"):
self.__thread_local.conn.close()
def __getitem__(self, model):
"""
DbModels (which we query) are accessed by using db as a dictionary
:param model: Subclass of zerodb.models.Model to return or create db entry for
:rtype: zerodb.db.DbModel
"""
# TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined)
if not issubclass(model, models.Model):
raise ModelException("Class <%s> is not a Model" % model.__name__)
if model not in self._models:
self._models[model] = DbModel(self, model)
return self._models[model]
def add(self, obj):
"""
Add newly created a Model object to the database
Stores *and* indexes it
:param zerodb.models.Model obj: Object to add to the database
:return: Added object's uid
:rtype: int
"""
if isinstance(obj, (list, set, tuple)):
return [self[o.__class__].add(o) for o in obj]
else:
return self[obj.__class__].add(obj)
def remove(self, obj):
"""
Remove existing object from the database + unindex it
:param zerodb.models.Model obj: Object to add to the database
"""
if isinstance(obj, models.Model):
self[obj.__class__].remove(obj)
return 1
elif hasattr(obj, "__iter__"):
ctr = 0
for o in obj:
ctr += 1
self[o.__class__].remove(o)
return ctr
else:
raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__)
class DB(object):
"""
Database for this user. Everything is used through this class
"""
db_factory = subdb.DB
auth_module = elliptic
encrypter = AES256Encrypter
compressor = None
def __init__(self, sock, username=None, password=None, realm="ZERO", debug=False, pool_timeout=3600, pool_size=7, **kw):
"""
:param str sock: UNIX (str) or TCP ((str, int)) socket
:type sock: str or tuple
:param str username: Username. Derived from password if not set
:param str password: Password or seed for private key
:param str realm: ZODB's realm
:param bool debug: Whether to log debug messages
"""
# ZODB doesn't like unicode here
username = username and str(username)
password = str(password)
if isinstance(sock, basestring):
sock = str(sock)
elif type(sock) in (list, tuple):
assert len(sock) == 2
sock = str(sock[0]), int(sock[1])
self.auth_module = kw.pop("auth_module", self.auth_module)
self.auth_module.register_auth()
if not username:
username = sha256("username" + sha256(password).digest()).digest()
self._init_default_crypto(passphrase=password)
# Store all the arguments necessary for login in this instance
self.__storage_kwargs = {
"sock": sock,
"username": username,
"password": password,
"realm": realm,
"debug": debug}
self.__db_kwargs = {
"pool_size": pool_size,
"pool_timeout": pool_timeout,
"cache_size": 50000,
"cache_size_bytes": 2 ** 30}
self.__db_kwargs.update(kw)
# For multi-threading
self.__pid = os.getpid()
self._init_db()
self._models = {}
def _init_default_crypto(self, passphrase=None):
if self.encrypter:
self.encrypter.register_class(default=True)
if self.compressor:
self.compressor.register(default=True)
init_crypto(passphrase=passphrase)
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = DB.db_factory(self._storage, **self.__db_kwargs)
self._conn_open()
def _conn_open(self):
"""Opens db connection and registers a destuction callback"""
self.__thread_local.conn = conn = self._db.open()
def destructor(conn_id):
conn = self._db.pool.all.data.get(conn_id, None)
if conn:
# Hack to make it not closing TM which is already removed
conn.opened = 0
conn.close()
self.__thread_watcher.watch(destructor, id(conn))
@property
def _root(self):
"""Access database root for this user"""
if os.getpid() != self.__pid:
# If a new process spins up, we need to re-initialize everything
self.__pid = os.getpid()
self._init_db()
else:
# Open connections from the pool when new threads spin up
# Should be closed when old thread-locals get garbage collected
if not hasattr(self.__thread_local, "conn") or\
self.__thread_local.conn.opened is None:
self._conn_open()
return self.__thread_local.conn.root()
@property
def _connection(self):
return self.__thread_local.conn
def disconnect(self):
if hasattr(self.__thread_local, "conn"):
self.__thread_local.conn.close()
def __getitem__(self, model):
"""
DbModels (which we query) are accessed by using db as a dictionary
:param model: Subclass of zerodb.models.Model to return or create db entry for
:rtype: zerodb.db.DbModel
"""
# TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined)
if not issubclass(model, models.Model):
raise ModelException("Class <%s> is not a Model" % model.__name__)
if model not in self._models:
self._models[model] = DbModel(self, model)
return self._models[model]
def add(self, obj):
"""
Add newly created a Model object to the database
Stores *and* indexes it
:param zerodb.models.Model obj: Object to add to the database
:return: Added object's uid
:rtype: int
"""
if isinstance(obj, (list, set, tuple)):
return [self[o.__class__].add(o) for o in obj]
else:
return self[obj.__class__].add(obj)
def remove(self, obj):
"""
Remove existing object from the database + unindex it
:param zerodb.models.Model obj: Object to add to the database
"""
self[obj.__class__].remove(obj)
class DB(object):
"""
Database for this user. Everything is used through this class
"""
encrypter = [AES256Encrypter, AES256EncrypterV0]
appname = "zerodb.com"
compressor = None
def __init__(
self,
sock,
key=None,
username=None,
password=None,
cert_file=None,
key_file=None,
server_cert=None,
security=None,
debug=False,
pool_timeout=3600,
pool_size=7,
autoreindex=True,
wait_timeout=30,
**kw
):
"""
:param str sock: UNIX (str) or TCP ((str, int)) socket
:type sock: str or tuple
:param str username: Username
:param str password: Password
:param bytes key: Encryption key
:param cert_file: Client certificate for authentication (pem)
:param key_file: Private key for that certificate (pem)
:param server_cert: Server certitificate if not registered with CA
:param function security: Key derivation function from
zerodb.crypto.kdf
:param bool debug: Whether to log debug messages
"""
if (cert_file or key_file) and not (cert_file and key_file):
raise TypeError("If you specify a cert file or a key file," " you must specify both.")
ssl_context = make_ssl(cert_file, key_file, server_cert)
if security is None:
security = kdf.guess(username, password, key_file, cert_file, self.appname, key)
password, key = security(username, password, key_file, cert_file, self.appname, key)
if password:
credentials = dict(name=username, password=password)
else:
credentials = None
if isinstance(sock, six.string_types):
sock = str(sock)
elif type(sock) in (list, tuple):
assert len(sock) == 2
sock = str(sock[0]), int(sock[1])
self._autoreindex = autoreindex
self._reindex_queue_processor = AutoReindexQueueProcessor(self, enabled=autoreindex)
component.provideUtility(self._reindex_queue_processor, IIndexQueueProcessor, "zerodb-indexer")
self._init_default_crypto(key=key)
# Store all the arguments necessary for login in this instance
self.__storage_kwargs = {
"sock": sock,
"ssl": ssl_context,
"cache_size": 2 ** 30,
"debug": debug,
"wait_timeout": wait_timeout,
"credentials": credentials,
}
self.__db_kwargs = {
"pool_size": pool_size,
"pool_timeout": pool_timeout,
"cache_size": 1000000,
"cache_size_bytes": 100 * 2 ** 20,
}
self.__db_kwargs.update(kw)
# For multi-threading
self.__pid = os.getpid()
self._init_db()
self._models = {}
@classmethod
def _init_default_crypto(self, **kw):
encrypters = self.encrypter
if not isinstance(encrypters, (list, tuple)):
encrypters = [self.encrypter]
elif not encrypters:
encrypters = []
if encrypters:
encrypters[0].register_class(default=True)
for e in encrypters[1:]:
e.register_class(default=False)
if self.compressor:
self.compressor.register(default=True)
init_crypto(**kw)
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
if self._autoreindex:
subscribers.init()
Random.atfork()
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = SubDB(self._storage, **self.__db_kwargs)
self._conn_open()
def _conn_open(self):
"""Opens db connection and registers a destuction callback"""
self.__thread_local.conn = conn = self._db.open()
def destructor(conn_id):
conn = self._db.pool.all.data.get(conn_id, None)
if conn:
# Hack to make it not closing TM which is already removed
conn.opened = 0
conn.close()
self.__thread_watcher.watch(destructor, id(conn))
@property
def _root(self):
"""Access database root for this user"""
if os.getpid() != self.__pid:
# If a new process spins up, we need to re-initialize everything
self.__pid = os.getpid()
self._init_db()
else:
# Open connections from the pool when new threads spin up
# Should be closed when old thread-locals get garbage collected
if not hasattr(self.__thread_local, "conn") or self.__thread_local.conn.opened is None:
self._conn_open()
return self.__thread_local.conn.root()
@property
def _connection(self):
return self.__thread_local.conn
def disconnect(self):
if hasattr(self.__thread_local, "conn"):
self.__thread_local.conn.close()
def __getitem__(self, model):
"""
DbModels (which we query) are accessed by using db as a dictionary
:param model: Subclass of zerodb.models.Model to return or create db entry for
:rtype: zerodb.db.DbModel
"""
# TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined)
if not issubclass(model, models.Model):
raise ModelException("Class <%s> is not a Model" % model.__name__)
if model not in self._models:
self._models[model] = DbModel(self, model)
return self._models[model]
def add(self, obj):
"""
Add newly created a Model object to the database
Stores *and* indexes it
:param zerodb.models.Model obj: Object to add to the database
:return: Added object's uid
:rtype: int
"""
if isinstance(obj, (list, set, tuple)):
return [self[o.__class__].add(o) for o in obj]
else:
return self[obj.__class__].add(obj)
def remove(self, obj):
"""
Remove existing object from the database + unindex it
:param zerodb.models.Model obj: Object to add to the database
"""
if isinstance(obj, models.Model):
self[obj.__class__].remove(obj)
return 1
elif hasattr(obj, "__iter__"):
ctr = 0
for o in obj:
ctr += 1
self[o.__class__].remove(o)
return ctr
else:
raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__)
def reindex(self, obj, attributes=None):
"""
Reindex one or multiple objects in the database
:param obj: Object to add to the database or its uid, or list of objects or uids
:type obj: zerodb.models.Model, list
:param attributes: Attributes of obj to be reindex
:type attributes: tuple, list
"""
if isinstance(obj, models.Model):
self[obj.__class__].reindex_one(obj, attributes)
elif isinstance(obj, (list, tuple, set, Sliceable)):
for o in obj:
assert isinstance(o, models.Model)
self[o.__class__].reindex_one(o, attributes)
else:
raise TypeError("ZeroDB object or list of these should be passed")
def pack(self):
"""
Remove old versions of objects
"""
self._db.pack()
def enableAutoReindex(self, enabled=True):
"""
Enable or disable auto reindex
"""
if enabled:
subscribers.init()
self._reindex_queue_processor.enabled = enabled
class DB(object):
"""
Database for this user. Everything is used through this class
"""
encrypter = [AES256Encrypter, AES256EncrypterV0]
appname = 'zerodb.com'
compressor = None
def __init__(self,
sock,
key=None,
username=None,
password=None,
cert_file=None,
key_file=None,
server_cert=None,
security=None,
debug=False,
pool_timeout=3600,
pool_size=7,
autoreindex=True,
wait_timeout=30,
**kw):
"""
:param str sock: UNIX (str) or TCP ((str, int)) socket
:type sock: str or tuple
:param str username: Username
:param str password: Password
:param bytes key: Encryption key
:param cert_file: Client certificate for authentication (pem)
:param key_file: Private key for that certificate (pem)
:param server_cert: Server certitificate if not registered with CA
:param function security: Key derivation function from
zerodb.crypto.kdf
:param bool debug: Whether to log debug messages
"""
if (cert_file or key_file) and not (cert_file and key_file):
raise TypeError("If you specify a cert file or a key file,"
" you must specify both.")
ssl_context = make_ssl(cert_file, key_file, server_cert)
if security is None:
security = kdf.guess(username, password, key_file, cert_file,
self.appname, key)
password, key = security(username, password, key_file, cert_file,
self.appname, key)
if password:
credentials = dict(name=username, password=password)
else:
credentials = None
if isinstance(sock, six.string_types):
sock = str(sock)
elif type(sock) in (list, tuple):
assert len(sock) == 2
sock = str(sock[0]), int(sock[1])
self._autoreindex = autoreindex
self._reindex_queue_processor = AutoReindexQueueProcessor(
self, enabled=autoreindex)
component.provideUtility(self._reindex_queue_processor,
IIndexQueueProcessor, 'zerodb-indexer')
self._init_default_crypto(key=key)
# Store all the arguments necessary for login in this instance
self.__storage_kwargs = {
"sock": sock,
"ssl": ssl_context,
"cache_size": 2**30,
"debug": debug,
"wait_timeout": wait_timeout,
"credentials": credentials,
}
self.__db_kwargs = {
"pool_size": pool_size,
"pool_timeout": pool_timeout,
"cache_size": 1000000,
"cache_size_bytes": 100 * 2**20
}
self.__db_kwargs.update(kw)
# For multi-threading
self.__pid = os.getpid()
self._init_db()
self._models = {}
@classmethod
def _init_default_crypto(self, **kw):
encrypters = self.encrypter
if not isinstance(encrypters, (list, tuple)):
encrypters = [self.encrypter]
elif not encrypters:
encrypters = []
if encrypters:
encrypters[0].register_class(default=True)
for e in encrypters[1:]:
e.register_class(default=False)
if self.compressor:
self.compressor.register(default=True)
init_crypto(**kw)
def _init_db(self):
"""We need this to be executed each time we are in a new process"""
if self._autoreindex:
subscribers.init()
Random.atfork()
self.__conn_refs = {}
self.__thread_local = threading.local()
self.__thread_watcher = ThreadWatcher()
self._storage = client_storage(**self.__storage_kwargs)
self._db = SubDB(self._storage, **self.__db_kwargs)
self._conn_open()
def _conn_open(self):
"""Opens db connection and registers a destuction callback"""
self.__thread_local.conn = conn = self._db.open()
def destructor(conn_id):
conn = self._db.pool.all.data.get(conn_id, None)
if conn:
# Hack to make it not closing TM which is already removed
conn.opened = 0
conn.close()
self.__thread_watcher.watch(destructor, id(conn))
@property
def _root(self):
"""Access database root for this user"""
if os.getpid() != self.__pid:
# If a new process spins up, we need to re-initialize everything
self.__pid = os.getpid()
self._init_db()
else:
# Open connections from the pool when new threads spin up
# Should be closed when old thread-locals get garbage collected
if not hasattr(self.__thread_local, "conn") or\
self.__thread_local.conn.opened is None:
self._conn_open()
return self.__thread_local.conn.root()
@property
def _connection(self):
return self.__thread_local.conn
def disconnect(self):
if hasattr(self.__thread_local, "conn"):
self.__thread_local.conn.close()
def __getitem__(self, model):
"""
DbModels (which we query) are accessed by using db as a dictionary
:param model: Subclass of zerodb.models.Model to return or create db entry for
:rtype: zerodb.db.DbModel
"""
# TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined)
if not issubclass(model, models.Model):
raise ModelException("Class <%s> is not a Model" % model.__name__)
if model not in self._models:
self._models[model] = DbModel(self, model)
return self._models[model]
def add(self, obj):
"""
Add newly created a Model object to the database
Stores *and* indexes it
:param zerodb.models.Model obj: Object to add to the database
:return: Added object's uid
:rtype: int
"""
if isinstance(obj, (list, set, tuple)):
return [self[o.__class__].add(o) for o in obj]
else:
return self[obj.__class__].add(obj)
def remove(self, obj):
"""
Remove existing object from the database + unindex it
:param zerodb.models.Model obj: Object to add to the database
"""
if isinstance(obj, models.Model):
self[obj.__class__].remove(obj)
return 1
elif hasattr(obj, "__iter__"):
ctr = 0
for o in obj:
ctr += 1
self[o.__class__].remove(o)
return ctr
else:
raise ModelException("Class <%s> is not a Model or iterable" %
obj.__class__.__name__)
def reindex(self, obj, attributes=None):
"""
Reindex one or multiple objects in the database
:param obj: Object to add to the database or its uid, or list of objects or uids
:type obj: zerodb.models.Model, list
:param attributes: Attributes of obj to be reindex
:type attributes: tuple, list
"""
if isinstance(obj, models.Model):
self[obj.__class__].reindex_one(obj, attributes)
elif isinstance(obj, (list, tuple, set, Sliceable)):
for o in obj:
assert isinstance(o, models.Model)
self[o.__class__].reindex_one(o, attributes)
else:
raise TypeError("ZeroDB object or list of these should be passed")
def pack(self):
"""
Remove old versions of objects
"""
self._db.pack()
def enableAutoReindex(self, enabled=True):
"""
Enable or disable auto reindex
"""
if enabled:
subscribers.init()
self._reindex_queue_processor.enabled = enabled