def _init_db(self): """We need this to be executed each time we are in a new process""" self.__conn_refs = {} self.__thread_local = threading.local() self.__thread_watcher = ThreadWatcher() self._storage = client_storage(**self.__storage_kwargs) self._db = DB.db_factory(self._storage, **self.__db_kwargs) self._conn_open()
def _init_db(self): """We need this to be executed each time we are in a new process""" if self._autoreindex: subscribers.init() Random.atfork() self.__conn_refs = {} self.__thread_local = threading.local() self.__thread_watcher = ThreadWatcher() self._storage = client_storage(**self.__storage_kwargs) self._db = SubDB(self._storage, **self.__db_kwargs) self._conn_open()
class DB(object): """ Database for this user. Everything is used through this class """ db_factory = subdb.DB auth_module = elliptic encrypter = AES256Encrypter compressor = None def __init__(self, sock, username=None, password=None, realm="ZERO", debug=False, pool_timeout=3600, pool_size=7, **kw): """ :param str sock: UNIX (str) or TCP ((str, int)) socket :type sock: str or tuple :param str username: Username. Derived from password if not set :param str password: Password or seed for private key :param str realm: ZODB's realm :param bool debug: Whether to log debug messages """ # ZODB doesn't like unicode here username = username and str(username) password = str(password) if isinstance(sock, basestring): sock = str(sock) elif type(sock) in (list, tuple): assert len(sock) == 2 sock = str(sock[0]), int(sock[1]) self.auth_module = kw.pop("auth_module", self.auth_module) self.auth_module.register_auth() if not username: username = sha256("username" + sha256(password).digest()).digest() self._init_default_crypto(passphrase=password) # Store all the arguments necessary for login in this instance self.__storage_kwargs = { "sock": sock, "username": username, "password": password, "realm": realm, "debug": debug} self.__db_kwargs = { "pool_size": pool_size, "pool_timeout": pool_timeout, "cache_size": 50000, "cache_size_bytes": 2 ** 30} self.__db_kwargs.update(kw) # For multi-threading self.__pid = os.getpid() self._init_db() self._models = {} def _init_default_crypto(self, passphrase=None): if self.encrypter: self.encrypter.register_class(default=True) if self.compressor: self.compressor.register(default=True) init_crypto(passphrase=passphrase) def _init_db(self): """We need this to be executed each time we are in a new process""" self.__conn_refs = {} self.__thread_local = threading.local() self.__thread_watcher = ThreadWatcher() self._storage = client_storage(**self.__storage_kwargs) self._db = DB.db_factory(self._storage, **self.__db_kwargs) self._conn_open() def _conn_open(self): """Opens db connection and registers a destuction callback""" self.__thread_local.conn = conn = self._db.open() def destructor(conn_id): conn = self._db.pool.all.data.get(conn_id, None) if conn: # Hack to make it not closing TM which is already removed conn.opened = 0 conn.close() self.__thread_watcher.watch(destructor, id(conn)) @property def _root(self): """Access database root for this user""" if os.getpid() != self.__pid: # If a new process spins up, we need to re-initialize everything self.__pid = os.getpid() self._init_db() else: # Open connections from the pool when new threads spin up # Should be closed when old thread-locals get garbage collected if not hasattr(self.__thread_local, "conn") or\ self.__thread_local.conn.opened is None: self._conn_open() return self.__thread_local.conn.root() @property def _connection(self): return self.__thread_local.conn def disconnect(self): if hasattr(self.__thread_local, "conn"): self.__thread_local.conn.close() def __getitem__(self, model): """ DbModels (which we query) are accessed by using db as a dictionary :param model: Subclass of zerodb.models.Model to return or create db entry for :rtype: zerodb.db.DbModel """ # TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined) if not issubclass(model, models.Model): raise ModelException("Class <%s> is not a Model" % model.__name__) if model not in self._models: self._models[model] = DbModel(self, model) return self._models[model] def add(self, obj): """ Add newly created a Model object to the database Stores *and* indexes it :param zerodb.models.Model obj: Object to add to the database :return: Added object's uid :rtype: int """ if isinstance(obj, (list, set, tuple)): return [self[o.__class__].add(o) for o in obj] else: return self[obj.__class__].add(obj) def remove(self, obj): """ Remove existing object from the database + unindex it :param zerodb.models.Model obj: Object to add to the database """ if isinstance(obj, models.Model): self[obj.__class__].remove(obj) return 1 elif hasattr(obj, "__iter__"): ctr = 0 for o in obj: ctr += 1 self[o.__class__].remove(o) return ctr else: raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__)
class DB(object): """ Database for this user. Everything is used through this class """ db_factory = subdb.DB auth_module = elliptic encrypter = AES256Encrypter compressor = None def __init__(self, sock, username=None, password=None, realm="ZERO", debug=False, pool_timeout=3600, pool_size=7, **kw): """ :param str sock: UNIX (str) or TCP ((str, int)) socket :type sock: str or tuple :param str username: Username. Derived from password if not set :param str password: Password or seed for private key :param str realm: ZODB's realm :param bool debug: Whether to log debug messages """ # ZODB doesn't like unicode here username = username and str(username) password = str(password) if isinstance(sock, basestring): sock = str(sock) elif type(sock) in (list, tuple): assert len(sock) == 2 sock = str(sock[0]), int(sock[1]) self.auth_module = kw.pop("auth_module", self.auth_module) self.auth_module.register_auth() if not username: username = sha256("username" + sha256(password).digest()).digest() self._init_default_crypto(passphrase=password) # Store all the arguments necessary for login in this instance self.__storage_kwargs = { "sock": sock, "username": username, "password": password, "realm": realm, "debug": debug} self.__db_kwargs = { "pool_size": pool_size, "pool_timeout": pool_timeout, "cache_size": 50000, "cache_size_bytes": 2 ** 30} self.__db_kwargs.update(kw) # For multi-threading self.__pid = os.getpid() self._init_db() self._models = {} def _init_default_crypto(self, passphrase=None): if self.encrypter: self.encrypter.register_class(default=True) if self.compressor: self.compressor.register(default=True) init_crypto(passphrase=passphrase) def _init_db(self): """We need this to be executed each time we are in a new process""" self.__conn_refs = {} self.__thread_local = threading.local() self.__thread_watcher = ThreadWatcher() self._storage = client_storage(**self.__storage_kwargs) self._db = DB.db_factory(self._storage, **self.__db_kwargs) self._conn_open() def _conn_open(self): """Opens db connection and registers a destuction callback""" self.__thread_local.conn = conn = self._db.open() def destructor(conn_id): conn = self._db.pool.all.data.get(conn_id, None) if conn: # Hack to make it not closing TM which is already removed conn.opened = 0 conn.close() self.__thread_watcher.watch(destructor, id(conn)) @property def _root(self): """Access database root for this user""" if os.getpid() != self.__pid: # If a new process spins up, we need to re-initialize everything self.__pid = os.getpid() self._init_db() else: # Open connections from the pool when new threads spin up # Should be closed when old thread-locals get garbage collected if not hasattr(self.__thread_local, "conn") or\ self.__thread_local.conn.opened is None: self._conn_open() return self.__thread_local.conn.root() @property def _connection(self): return self.__thread_local.conn def disconnect(self): if hasattr(self.__thread_local, "conn"): self.__thread_local.conn.close() def __getitem__(self, model): """ DbModels (which we query) are accessed by using db as a dictionary :param model: Subclass of zerodb.models.Model to return or create db entry for :rtype: zerodb.db.DbModel """ # TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined) if not issubclass(model, models.Model): raise ModelException("Class <%s> is not a Model" % model.__name__) if model not in self._models: self._models[model] = DbModel(self, model) return self._models[model] def add(self, obj): """ Add newly created a Model object to the database Stores *and* indexes it :param zerodb.models.Model obj: Object to add to the database :return: Added object's uid :rtype: int """ if isinstance(obj, (list, set, tuple)): return [self[o.__class__].add(o) for o in obj] else: return self[obj.__class__].add(obj) def remove(self, obj): """ Remove existing object from the database + unindex it :param zerodb.models.Model obj: Object to add to the database """ self[obj.__class__].remove(obj)
class DB(object): """ Database for this user. Everything is used through this class """ encrypter = [AES256Encrypter, AES256EncrypterV0] appname = "zerodb.com" compressor = None def __init__( self, sock, key=None, username=None, password=None, cert_file=None, key_file=None, server_cert=None, security=None, debug=False, pool_timeout=3600, pool_size=7, autoreindex=True, wait_timeout=30, **kw ): """ :param str sock: UNIX (str) or TCP ((str, int)) socket :type sock: str or tuple :param str username: Username :param str password: Password :param bytes key: Encryption key :param cert_file: Client certificate for authentication (pem) :param key_file: Private key for that certificate (pem) :param server_cert: Server certitificate if not registered with CA :param function security: Key derivation function from zerodb.crypto.kdf :param bool debug: Whether to log debug messages """ if (cert_file or key_file) and not (cert_file and key_file): raise TypeError("If you specify a cert file or a key file," " you must specify both.") ssl_context = make_ssl(cert_file, key_file, server_cert) if security is None: security = kdf.guess(username, password, key_file, cert_file, self.appname, key) password, key = security(username, password, key_file, cert_file, self.appname, key) if password: credentials = dict(name=username, password=password) else: credentials = None if isinstance(sock, six.string_types): sock = str(sock) elif type(sock) in (list, tuple): assert len(sock) == 2 sock = str(sock[0]), int(sock[1]) self._autoreindex = autoreindex self._reindex_queue_processor = AutoReindexQueueProcessor(self, enabled=autoreindex) component.provideUtility(self._reindex_queue_processor, IIndexQueueProcessor, "zerodb-indexer") self._init_default_crypto(key=key) # Store all the arguments necessary for login in this instance self.__storage_kwargs = { "sock": sock, "ssl": ssl_context, "cache_size": 2 ** 30, "debug": debug, "wait_timeout": wait_timeout, "credentials": credentials, } self.__db_kwargs = { "pool_size": pool_size, "pool_timeout": pool_timeout, "cache_size": 1000000, "cache_size_bytes": 100 * 2 ** 20, } self.__db_kwargs.update(kw) # For multi-threading self.__pid = os.getpid() self._init_db() self._models = {} @classmethod def _init_default_crypto(self, **kw): encrypters = self.encrypter if not isinstance(encrypters, (list, tuple)): encrypters = [self.encrypter] elif not encrypters: encrypters = [] if encrypters: encrypters[0].register_class(default=True) for e in encrypters[1:]: e.register_class(default=False) if self.compressor: self.compressor.register(default=True) init_crypto(**kw) def _init_db(self): """We need this to be executed each time we are in a new process""" if self._autoreindex: subscribers.init() Random.atfork() self.__conn_refs = {} self.__thread_local = threading.local() self.__thread_watcher = ThreadWatcher() self._storage = client_storage(**self.__storage_kwargs) self._db = SubDB(self._storage, **self.__db_kwargs) self._conn_open() def _conn_open(self): """Opens db connection and registers a destuction callback""" self.__thread_local.conn = conn = self._db.open() def destructor(conn_id): conn = self._db.pool.all.data.get(conn_id, None) if conn: # Hack to make it not closing TM which is already removed conn.opened = 0 conn.close() self.__thread_watcher.watch(destructor, id(conn)) @property def _root(self): """Access database root for this user""" if os.getpid() != self.__pid: # If a new process spins up, we need to re-initialize everything self.__pid = os.getpid() self._init_db() else: # Open connections from the pool when new threads spin up # Should be closed when old thread-locals get garbage collected if not hasattr(self.__thread_local, "conn") or self.__thread_local.conn.opened is None: self._conn_open() return self.__thread_local.conn.root() @property def _connection(self): return self.__thread_local.conn def disconnect(self): if hasattr(self.__thread_local, "conn"): self.__thread_local.conn.close() def __getitem__(self, model): """ DbModels (which we query) are accessed by using db as a dictionary :param model: Subclass of zerodb.models.Model to return or create db entry for :rtype: zerodb.db.DbModel """ # TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined) if not issubclass(model, models.Model): raise ModelException("Class <%s> is not a Model" % model.__name__) if model not in self._models: self._models[model] = DbModel(self, model) return self._models[model] def add(self, obj): """ Add newly created a Model object to the database Stores *and* indexes it :param zerodb.models.Model obj: Object to add to the database :return: Added object's uid :rtype: int """ if isinstance(obj, (list, set, tuple)): return [self[o.__class__].add(o) for o in obj] else: return self[obj.__class__].add(obj) def remove(self, obj): """ Remove existing object from the database + unindex it :param zerodb.models.Model obj: Object to add to the database """ if isinstance(obj, models.Model): self[obj.__class__].remove(obj) return 1 elif hasattr(obj, "__iter__"): ctr = 0 for o in obj: ctr += 1 self[o.__class__].remove(o) return ctr else: raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__) def reindex(self, obj, attributes=None): """ Reindex one or multiple objects in the database :param obj: Object to add to the database or its uid, or list of objects or uids :type obj: zerodb.models.Model, list :param attributes: Attributes of obj to be reindex :type attributes: tuple, list """ if isinstance(obj, models.Model): self[obj.__class__].reindex_one(obj, attributes) elif isinstance(obj, (list, tuple, set, Sliceable)): for o in obj: assert isinstance(o, models.Model) self[o.__class__].reindex_one(o, attributes) else: raise TypeError("ZeroDB object or list of these should be passed") def pack(self): """ Remove old versions of objects """ self._db.pack() def enableAutoReindex(self, enabled=True): """ Enable or disable auto reindex """ if enabled: subscribers.init() self._reindex_queue_processor.enabled = enabled
class DB(object): """ Database for this user. Everything is used through this class """ encrypter = [AES256Encrypter, AES256EncrypterV0] appname = 'zerodb.com' compressor = None def __init__(self, sock, key=None, username=None, password=None, cert_file=None, key_file=None, server_cert=None, security=None, debug=False, pool_timeout=3600, pool_size=7, autoreindex=True, wait_timeout=30, **kw): """ :param str sock: UNIX (str) or TCP ((str, int)) socket :type sock: str or tuple :param str username: Username :param str password: Password :param bytes key: Encryption key :param cert_file: Client certificate for authentication (pem) :param key_file: Private key for that certificate (pem) :param server_cert: Server certitificate if not registered with CA :param function security: Key derivation function from zerodb.crypto.kdf :param bool debug: Whether to log debug messages """ if (cert_file or key_file) and not (cert_file and key_file): raise TypeError("If you specify a cert file or a key file," " you must specify both.") ssl_context = make_ssl(cert_file, key_file, server_cert) if security is None: security = kdf.guess(username, password, key_file, cert_file, self.appname, key) password, key = security(username, password, key_file, cert_file, self.appname, key) if password: credentials = dict(name=username, password=password) else: credentials = None if isinstance(sock, six.string_types): sock = str(sock) elif type(sock) in (list, tuple): assert len(sock) == 2 sock = str(sock[0]), int(sock[1]) self._autoreindex = autoreindex self._reindex_queue_processor = AutoReindexQueueProcessor( self, enabled=autoreindex) component.provideUtility(self._reindex_queue_processor, IIndexQueueProcessor, 'zerodb-indexer') self._init_default_crypto(key=key) # Store all the arguments necessary for login in this instance self.__storage_kwargs = { "sock": sock, "ssl": ssl_context, "cache_size": 2**30, "debug": debug, "wait_timeout": wait_timeout, "credentials": credentials, } self.__db_kwargs = { "pool_size": pool_size, "pool_timeout": pool_timeout, "cache_size": 1000000, "cache_size_bytes": 100 * 2**20 } self.__db_kwargs.update(kw) # For multi-threading self.__pid = os.getpid() self._init_db() self._models = {} @classmethod def _init_default_crypto(self, **kw): encrypters = self.encrypter if not isinstance(encrypters, (list, tuple)): encrypters = [self.encrypter] elif not encrypters: encrypters = [] if encrypters: encrypters[0].register_class(default=True) for e in encrypters[1:]: e.register_class(default=False) if self.compressor: self.compressor.register(default=True) init_crypto(**kw) def _init_db(self): """We need this to be executed each time we are in a new process""" if self._autoreindex: subscribers.init() Random.atfork() self.__conn_refs = {} self.__thread_local = threading.local() self.__thread_watcher = ThreadWatcher() self._storage = client_storage(**self.__storage_kwargs) self._db = SubDB(self._storage, **self.__db_kwargs) self._conn_open() def _conn_open(self): """Opens db connection and registers a destuction callback""" self.__thread_local.conn = conn = self._db.open() def destructor(conn_id): conn = self._db.pool.all.data.get(conn_id, None) if conn: # Hack to make it not closing TM which is already removed conn.opened = 0 conn.close() self.__thread_watcher.watch(destructor, id(conn)) @property def _root(self): """Access database root for this user""" if os.getpid() != self.__pid: # If a new process spins up, we need to re-initialize everything self.__pid = os.getpid() self._init_db() else: # Open connections from the pool when new threads spin up # Should be closed when old thread-locals get garbage collected if not hasattr(self.__thread_local, "conn") or\ self.__thread_local.conn.opened is None: self._conn_open() return self.__thread_local.conn.root() @property def _connection(self): return self.__thread_local.conn def disconnect(self): if hasattr(self.__thread_local, "conn"): self.__thread_local.conn.close() def __getitem__(self, model): """ DbModels (which we query) are accessed by using db as a dictionary :param model: Subclass of zerodb.models.Model to return or create db entry for :rtype: zerodb.db.DbModel """ # TODO implement list of keys, writing to arbitrary (new) dbmodel (which is not defined) if not issubclass(model, models.Model): raise ModelException("Class <%s> is not a Model" % model.__name__) if model not in self._models: self._models[model] = DbModel(self, model) return self._models[model] def add(self, obj): """ Add newly created a Model object to the database Stores *and* indexes it :param zerodb.models.Model obj: Object to add to the database :return: Added object's uid :rtype: int """ if isinstance(obj, (list, set, tuple)): return [self[o.__class__].add(o) for o in obj] else: return self[obj.__class__].add(obj) def remove(self, obj): """ Remove existing object from the database + unindex it :param zerodb.models.Model obj: Object to add to the database """ if isinstance(obj, models.Model): self[obj.__class__].remove(obj) return 1 elif hasattr(obj, "__iter__"): ctr = 0 for o in obj: ctr += 1 self[o.__class__].remove(o) return ctr else: raise ModelException("Class <%s> is not a Model or iterable" % obj.__class__.__name__) def reindex(self, obj, attributes=None): """ Reindex one or multiple objects in the database :param obj: Object to add to the database or its uid, or list of objects or uids :type obj: zerodb.models.Model, list :param attributes: Attributes of obj to be reindex :type attributes: tuple, list """ if isinstance(obj, models.Model): self[obj.__class__].reindex_one(obj, attributes) elif isinstance(obj, (list, tuple, set, Sliceable)): for o in obj: assert isinstance(o, models.Model) self[o.__class__].reindex_one(o, attributes) else: raise TypeError("ZeroDB object or list of these should be passed") def pack(self): """ Remove old versions of objects """ self._db.pack() def enableAutoReindex(self, enabled=True): """ Enable or disable auto reindex """ if enabled: subscribers.init() self._reindex_queue_processor.enabled = enabled