Example #1
0
class AbsddbStorage(object):
    
    implements(interface.IStorage)
    
    def __init__(self, entity, env, factory):
        self.entity = IEntity(entity)
        self.factory = Factory(self.entity)
        self.storageFactory = interface.IStorageFactory(factory)
        self.unpacker = EntityUnpacker(self.entity)
        
        self.id_cache = {}
        
        self.env = env
        self.pdb = adb.ADB(self.env)
        
        self.fields_unpackers = {}
        self.fields_storages = {}
        for field in filter(lambda field: IEntity.providedBy(field.type), self.entity.fields()):
            self.fields_unpackers[field.name] = EntityUnpacker(field.type)
        
        self.idb = {}
        for field in self.entity.indexed_fields():
            self.idb[field.name] = adb.ADB(self.env)
        
        self.initialized = False
        self.initd = None
    
    def init(self, *args):
        
        if self.initialized:
            return defer.succeed(True)
        
        if self.initd is not None:
            return self.initd
        
        def open_pdb(_):
            return self.pdb.open(
                                 "%s.db" % self.entity.__name__,
                                 self.entity.__name__,
                                 db.DB_HASH,
                                 db.DB_CREATE | db.DB_AUTO_COMMIT
                                 )
        
        def set_idb_flags(_):
            dlist = map(lambda name: self.idb[name].set_flags(db.DB_DUP | db.DB_DUPSORT), self.idb.keys())
            return defer.DeferredList(dlist, fireOnOneErrback = True)
        
        def open_idb(_):
            dlist = map(
                        lambda name: self.idb[name].open(
                                                         "%s.idx" % self.entity.__name__,
                                                         name,
                                                         db.DB_BTREE,
                                                         db.DB_CREATE | db.DB_AUTO_COMMIT
                                                         ),
                        self.idb.keys()
                        )
            
            return defer.DeferredList(dlist, fireOnOneErrback = True)
        
        def associate(_):
            
            def create_callback(name):
                def callback(key, data):
                    values = pickle.loads(data)
                    return pickle.dumps(values[name])
                return callback
            
            dlist = map(lambda name: self.pdb.associate(self.idb[name], create_callback(name), db.DB_CREATE), self.idb.keys())
            return defer.DeferredList(dlist, fireOnOneErrback = True)
        
        def get_storages(_):
            
            def get_storage(field):
                
                def set_storage(storage):
                    self.fields_storages[field.name] = storage
                    return defer.succeed(storage)
                
                d = self.storageFactory.get(field.type)
                d.addCallback(set_storage)
                return d
            
            fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.fields())
            dlist = map(lambda field: get_storage(field), fields)
            return defer.DeferredList(dlist, fireOnOneErrback = True)
        
        def complete(_):
            self.initialized = True
            return defer.succeed(True)
        
        self.initd = defer.Deferred()
        
        self.initd.addCallback(open_pdb)
        self.initd.addCallback(set_idb_flags)
        self.initd.addCallback(open_idb)
        self.initd.addCallback(associate)
        self.initd.addCallback(get_storages)
        self.initd.addCallback(complete)
        
        reactor.callLater(0, self.initd.callback, None)
        return self.initd
    
    def put(self, instance):
        
        def unpack(_):
                
            def unpack_identity(instance):
                return self.unpacker.unpack_identity(instance)
                
            def unpack_indexes(instance):
                    
                def complete(indexes):
                        
                    fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.indexed_fields())
                        
                    if len(fields):
                            
                        def complete(results):
                            for (field, (success, value)) in map(lambda field, result: (field, result), fields, results):
                                if not success:
                                    return defer.fail(value)
                                indexes[field.name] = value
                            return defer.succeed(indexes)
                            
                        dlist = map(lambda field: self.fields_unpackers[field.name].unpack_identity(indexes[field.name]), fields)
                        d = defer.DeferredList(dlist, consumeErrors = True)
                        d.addCallback(complete)
                        return d
                        
                    else:
                        return defer.succeed(indexes)
                    
                d = self.unpacker.unpack_indexed(instance)
                d.addCallback(complete)
                return d
                
            def unpack_required(instance):
                    
                def complete(required):
                        
                    fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.required_fields())
                        
                    if len(fields):
                            
                        def complete(results):
                            for (field, (success, value)) in map(lambda field, result: (field, result), fields, results):
                                if not success:
                                    return defer.fail(value)
                                required[field.name] = value
                            return defer.succeed(required)
                            
                        dlist = map(lambda field: self.fields_unpackers[field.name].unpack_identity(required[field.name]), fields)
                        d = defer.DeferredList(dlist, consumeErrors = True)
                        d.addCallback(complete)
                        return d
                        
                    else:
                        return defer.succeed(required) 
                        
                d = self.unpacker.unpack_required(instance)
                d.addCallback(complete)
                return d
                
            def unpack_optional(instance):
                    
                def complete(optional):
                        
                    fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.optional_fields())
                        
                    if len(fields):
                            
                        def complete(results):
                            for (field, (success, value)) in map(lambda field, result: (field, result), fields, results):
                                if not success:
                                    return defer.fail(value)
                                optional[field.name] = value
                            return defer.succeed(optional)
                            
                        dlist = map(lambda field: self.fields_unpackers[field.name].unpack_identity(optional[field.name]), fields)
                        d = defer.DeferredList(dlist, consumeErrors = True)
                        d.addCallback(complete)
                        return d
                            
                    else:
                        return defer.succeed(optional)
                        
                d = self.unpacker.unpack_optional(instance)
                d.addCallback(complete)
                return d
                
            def complete(results):
                rs = ()
                for (success, result) in results:
                    if not success:
                        return defer.fail(result)
                    rs += (result,)
                return defer.succeed(rs)
                
            d = defer.DeferredList([
                                    unpack_identity(instance),
                                    unpack_indexes(instance),
                                    unpack_required(instance),
                                    unpack_optional(instance)
                                    ], consumeErrors = True)
            d.addCallback(complete)
            return d
            
        def put((identity, indexes, required, optional)):
            
            def get_fields((identity, indexes, required, optional)):
                fields = dict(required.items() + optional.items())
                return defer.succeed((identity, fields))
                
            def put_instance((identity, fields)):
                data = pickle.dumps(fields)
                return self.pdb.put(identity, data)
                
            def complete(results):
                return defer.succeed(instance)
                
            d = defer.Deferred()
            
            d.addCallback(get_fields)
            d.addCallback(put_instance)
            d.addCallback(complete)
            
            reactor.callLater(0, d.callback, (identity, indexes, required, optional))
            return d
            
        def complete(result):
            return defer.succeed(instance)
            
        d = defer.Deferred()
        
        d.addCallback(self.init)    
        d.addCallback(unpack)
        d.addCallback(put)
        d.addCallback(complete)
            
        reactor.callLater(0, d.callback, instance)
        return d
    
    def remove(self, instance):
        
        def unpack_identity(_):
            return self.unpacker.unpack_identity(instance)
        
        def check_exists(identity):
            
            def complete(result):
                if result:
                    return defer.succeed(identity)
                return defer.fail(AttributeError("Entity not exists: %s" % identity))
            
            d = self.pdb.exists(identity)
            d.addCallback(complete)
            return d
        
        def remove_from_cache(identity):
            if self.id_cache.has_key(identity):
                del self.id_cache[identity]
            return defer.succeed(identity)
        
        def remove_from_pdb(identity):
            return self.pdb.delete(identity)
        
        def complete(_):
            return defer.succeed(True)
        
        d = defer.Deferred()
        
        d.addCallback(self.init)
        d.addCallback(unpack_identity)
        d.addCallback(check_exists)
        d.addCallback(remove_from_cache)
        d.addCallback(remove_from_pdb)
        d.addCallback(complete)
        
        reactor.callLater(0, d.callback, None)
        return d
    
    def get(self, identity):
        
        def check_exists(_):
            
            def complete(result):
                if result:
                    return defer.succeed(identity)
                return defer.fail(AttributeError("Entity not exists: %s" % identity))
            
            d = self.pdb.exists(identity)
            d.addCallback(complete)
            return d
        
        def get(identity):
            
            if self.id_cache.has_key(identity):
                return defer.succeed((self.id_cache[identity], identity))
            
            def unpack_fields(identity):
                
                def fetch(identity):
    
                    def complete(data):
                        items = pickle.loads(data)
                        return defer.succeed((identity, items))
                    
                    d = self.pdb.get(identity)
                    d.addCallback(complete)
                    return d
                
                def complete((identity, items)):
                
                    fields = {}
                    toFetch = {}
        
                    for (name, value) in items.items():
                        if self.fields_storages.has_key(name):
                            fields[name] = None
                            toFetch[name] = value
                        else:
                            fields[name] = value
                        
                    return defer.succeed((identity, fields, toFetch))
                
                d = defer.Deferred()
                d.addCallback(fetch)
                d.addCallback(complete)
                
                reactor.callLater(0, d.callback, identity)
                return d
                
            
            def create((identity, fields, toFetch)):
                
                def create((identity, fields, toFetch)):
        
                    def complete(instance):
                        setattr(instance, self.entity.identity().name, identity)
                        return defer.succeed((instance, identity, toFetch))
                    
                    d = self.factory.create(**fields)
                    d.addCallback(complete)
                    return d
                
                d = defer.Deferred()
                d.addCallback(create)
                reactor.callLater(0, d.callback, (identity, fields, toFetch))
                return d
            
            def add_to_cache((instance, identity, toFetch)):
                self.id_cache[identity] = instance
                return defer.succeed((instance, identity, toFetch))
            
            def fetch((instance, identity, toFetch)):
                
                if not len(toFetch):
                    return defer.succeed((instance, identity, {}))
                
                def complete(results):
                    fetched = {}
                    for (name, (success, value)) in map(lambda field, result: (field, result), fields, results):
                        fetched[name] = value
                    return defer.succeed((instance, identity, fetched))
                
                fields = toFetch.keys()
                dlist = map(lambda field: self.fields_storages[field].get(toFetch[field]), fields)
                d = defer.DeferredList(dlist, consumeErrors = True)
                d.addCallback(complete)
                return d
            
            def set_fetched((instance, identity, fetched)):
                for (name, value) in fetched.items():
                    setattr(instance, name, value)
                return defer.succeed((instance, identity))
            
            def complete((instance, identity)):
                return defer.succeed((instance, identity))
            
            d = defer.Deferred()
            
            d.addCallback(unpack_fields)
            d.addCallback(create)
            d.addCallback(add_to_cache)
            d.addCallback(fetch)
            d.addCallback(set_fetched)
            d.addCallback(complete)
            
            reactor.callLater(0, d.callback, identity)
            return d
        
        def complete((instance, identity)):
            return defer.succeed(instance)
        
        d = defer.Deferred()
        
        d.addCallback(self.init)
        d.addCallback(check_exists)
        d.addCallback(get)
        d.addCallback(complete)
        
        reactor.callLater(0, d.callback, identity)
        return d
    
    def where(self, **kwargs):
        
        def calc_index(_):
            return defer.succeed((kwargs.keys()[0], kwargs[kwargs.keys()[0]]))
        
        def get((index, value)):
            
            if Entity.providedBy(value):
                value = value.id
            
            if self.idb.has_key(index):
                
                def get_cursor(_):
                    return self.idb[index].cursor()
                
                def fetch(cursor):
                    
                    ids = []
                    
                    def set_key(_):
                        
                        def complete(result):
                            
                            if result is None:
                                return defer.succeed(False)
                            
                            if result[0] == pickle.dumps(value):
                                ids.append(result[1])
                                return defer.succeed(True)
                            
                            return defer.succeed(False)
                    
                        d = cursor.pget_(pickle.dumps(value), db.DB_SET_RANGE)
                        d.addCallback(complete)
                        return d
                    
                    def fetch_next(result):
                        
                        if not result:
                            return defer.succeed(True)
                        
                        def complete(result):
                            if result is not None:
                                ids.append(result[1])
                                return fetch_next(True)
                            return defer.succeed(True)
                        
                        d = cursor.pget_(pickle.dumps(value), db.DB_NEXT_DUP)
                        d.addCallback(complete)
                        return d
                    
                    def complete(_):
                        return defer.succeed((cursor, ids))
                    
                    d = defer.Deferred()
                    
                    d.addCallback(set_key)
                    d.addCallback(fetch_next)
                    d.addCallback(complete)
                    
                    reactor.callLater(0, d.callback, None)
                    return d
                
                def close_cursor((cursor, identities)):
                    
                    def complete(_):
                        return defer.succeed(identities)
                    
                    d = cursor.close()
                    d.addCallback(complete)
                    return d
                
                d = defer.Deferred()
                
                d.addCallback(get_cursor)
                d.addCallback(fetch)
                d.addCallback(close_cursor)
                
                reactor.callLater(0, d.callback, None)
                return d
                
            return defer.succeed([])
        
        def complete(identities):
            return defer.succeed(identities)
        
        d = defer.Deferred()
        
        d.addCallback(self.init)
        d.addCallback(calc_index)
        d.addCallback(get)
        d.addCallback(complete)
        
        reactor.callLater(0, d.callback, kwargs)
        return d
Example #2
0
class MemoryStorage(object):
    
    implements(interface.IStorage)
    
    def __init__(self, entity, factory):
        self.entity = IEntity(entity)
        self.factory = Factory(self.entity)
        self.storageFactory = interface.IStorageFactory(factory)
        self.unpacker = EntityUnpacker(self.entity)
        self.id_cache = {}
        self.id_map = {}
        self.index_map = {}
        self.fields_storages = {}
        self.fields_unpackers = {}
        for field in filter(lambda field: IEntity.providedBy(field.type), self.entity.fields()):
            self.fields_unpackers[field.name] = EntityUnpacker(field.type)
        for field in self.entity.indexed_fields():
            self.index_map[field.name] = {}
        self.initialized = False
        self.initd = None
    
    def init(self, *args):
        
        if self.initialized:
            return defer.succeed(True)
        
        if self.initd is not None:
            return self.initd
        
        def get_storages(_):
            
            def get_storage(field):
                
                def set_storage(storage):
                    self.fields_storages[field.name] = storage
                    return defer.succeed(storage)
                
                d = self.storageFactory.get(field.type)
                d.addCallback(set_storage)
                return d
            
            fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.fields())
            dlist = map(lambda field: get_storage(field), fields)
            return defer.DeferredList(dlist, fireOnOneErrback = True)
        
        def complete(_):
            self.initialized = True
            return defer.succeed(True)
        
        self.initd = defer.Deferred()
        self.initd.addCallback(get_storages)
        self.initd.addCallback(complete)
        reactor.callLater(0, self.initd.callback, None)
        return self.initd
            
    def put(self, instance):
            
        def unpack(_):
                
            def unpack_identity(instance):
                return self.unpacker.unpack_identity(instance)
                
            def unpack_indexes(instance):
                    
                def complete(indexes):
                        
                    fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.indexed_fields())
                        
                    if len(fields):
                            
                        def complete(results):
                            for (field, (success, value)) in map(lambda field, result: (field, result), fields, results):
                                if not success:
                                    return defer.fail(value)
                                indexes[field.name] = value
                            return defer.succeed(indexes)
                            
                        dlist = map(lambda field: self.fields_unpackers[field.name].unpack_identity(indexes[field.name]), fields)
                        d = defer.DeferredList(dlist, consumeErrors = True)
                        d.addCallback(complete)
                        return d
                        
                    else:
                        return defer.succeed(indexes)
                    
                d = self.unpacker.unpack_indexed(instance)
                d.addCallback(complete)
                return d
                
            def unpack_required(instance):
                    
                def complete(required):
                        
                    fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.required_fields())
                        
                    if len(fields):
                            
                        def complete(results):
                            for (field, (success, value)) in map(lambda field, result: (field, result), fields, results):
                                if not success:
                                    return defer.fail(value)
                                required[field.name] = value
                            return defer.succeed(required)
                            
                        dlist = map(lambda field: self.fields_unpackers[field.name].unpack_identity(required[field.name]), fields)
                        d = defer.DeferredList(dlist, consumeErrors = True)
                        d.addCallback(complete)
                        return d
                        
                    else:
                        return defer.succeed(required) 
                        
                d = self.unpacker.unpack_required(instance)
                d.addCallback(complete)
                return d
                
            def unpack_optional(instance):
                    
                def complete(optional):
                        
                    fields = filter(lambda field: IEntity.providedBy(field.type), self.entity.optional_fields())
                        
                    if len(fields):
                            
                        def complete(results):
                            for (field, (success, value)) in map(lambda field, result: (field, result), fields, results):
                                if not siccess:
                                    return defer.fail(value)
                                optional[field.name] = value
                            return defer.succeed(optional)
                            
                        dlist = map(lambda field: self.fields_unpackers[field.name].unpack_identity(optional[field.name]), fields)
                        d = defer.DeferredList(dlist, consumeErrors = True)
                        d.addCallback(complete)
                        return d
                            
                    else:
                        return defer.succeed(optional)
                        
                d = self.unpacker.unpack_optional(instance)
                d.addCallback(complete)
                return d
                
            def complete(results):
                rs = ()
                for (success, result) in results:
                    if not success:
                        return defer.fail(result)
                    rs += (result,)
                return defer.succeed(rs)
                
            d = defer.DeferredList([
                                    unpack_identity(instance),
                                    unpack_indexes(instance),
                                    unpack_required(instance),
                                    unpack_optional(instance)
                                    ], consumeErrors = True)
            d.addCallback(complete)
            return d
            
        def put((identity, indexes, required, optional)):
                
            def put_instance((identity, required, optional)):
                self.id_map[identity] = {"required" : required, "optional" : optional}
                return defer.succeed(True)
                
            def put_indexes((identity, indexes)):
                for index in indexes.keys():
                    if not self.index_map[index].has_key(indexes[index]):
                        self.index_map[index][indexes[index]] = {}
                    self.index_map[index][indexes[index]][identity] = 1
                return defer.succeed(True)
                
            def complete(results):
                return defer.succeed(instance)
                
            d = defer.DeferredList([
                                    put_instance((identity, required, optional)),
                                    put_indexes((identity, indexes))
                                    ], consumeErrors = True)
                
            d.addCallback(complete)
            return d
            
        def complete(result):
            return defer.succeed(instance)
            
        d = defer.Deferred()
        
        d.addCallback(self.init)    
        d.addCallback(unpack)
        d.addCallback(put)
        d.addCallback(complete)
            
        reactor.callLater(0, d.callback, None)
        return d
    
    def get(self, identity):
        
        def check_exists(_):
            if self.id_map.has_key(identity):
                return defer.succeed(identity)
            return defer.fail(AttributeError("Entity not exists: %s" % identity))
        
        def get(identity):
            
            if self.id_cache.has_key(identity):
                return defer.succeed((self.id_cache[identity], identity))
            
            def unpack_fields(identity):
                fields = {}
                toFetch = {}
        
                for (name, value) in self.id_map[identity]["required"].items():
                    if self.fields_storages.has_key(name):
                        fields[name] = None
                        toFetch[name] = value
                    else:
                        fields[name] = value
                        
                for (name, value) in self.id_map[identity]["optional"].items():
                    if self.fields_storages.has_key(name):
                        fields[name] = None
                        toFetch[name] = value
                    else:
                        fields[name] = value
       
                return defer.succeed((identity, fields, toFetch))
                
            def create((identity, fields, toFetch)):
                
                def create((identity, fields, toFetch)):
        
                    def complete(instance):
                        setattr(instance, self.entity.identity().name, identity)
                        return defer.succeed((instance, identity, toFetch))
                    
                    d = self.factory.create(**fields)
                    d.addCallback(complete)
                    return d
                
                d = defer.Deferred()
                d.addCallback(create)
                reactor.callLater(0, d.callback, (identity, fields, toFetch))
                return d
            
            def add_to_cache((instance, identity, toFetch)):
                self.id_cache[identity] = instance
                return defer.succeed((instance, identity, toFetch))
            
            def fetch((instance, identity, toFetch)):
                
                if not len(toFetch):
                    return defer.succeed((instance, identity, {}))
                
                def complete(results):
                    fetched = {}
                    for (name, (success, value)) in map(lambda field, result: (field, result), fields, results):
                        fetched[name] = value
                    return defer.succeed((instance, identity, fetched))
                
                fields = toFetch.keys()
                dlist = map(lambda field: self.fields_storages[field].get(toFetch[field]), fields)
                d = defer.DeferredList(dlist, consumeErrors = True)
                d.addCallback(complete)
                return d
            
            def set_fetched((instance, identity, fetched)):
                for (name, value) in fetched.items():
                    setattr(instance, name, value)
                return defer.succeed((instance, identity))
            
            def complete((instance, identity)):
                return defer.succeed((instance, identity))
            
            d = defer.Deferred()
            
            d.addCallback(unpack_fields)
            d.addCallback(create)
            d.addCallback(add_to_cache)
            d.addCallback(fetch)
            d.addCallback(set_fetched)
            d.addCallback(complete)
            
            reactor.callLater(0, d.callback, identity)
            return d
        
        def complete((instance, identity)):
            return defer.succeed(instance)
        
        d = defer.Deferred()
        
        d.addCallback(self.init)
        d.addCallback(check_exists)
        d.addCallback(get)
        d.addCallback(complete)
        
        reactor.callLater(0, d.callback, None)
        return d
    
    def where(self, **kwargs):
        
        def calc_index(_):
            return defer.succeed((kwargs.keys()[0], kwargs[kwargs.keys()[0]]))
        
        def get((index, value)):
            
            if Entity.providedBy(value):
                value = value.id
            
            if self.index_map.has_key(index):
                if self.index_map[index].has_key(value):
                    return defer.succeed(self.index_map[index][value].keys())
            return []
        
        def complete(identities):
            return defer.succeed(identities)
        
        d = defer.Deferred()
        
        d.addCallback(self.init)
        d.addCallback(calc_index)
        d.addCallback(get)
        d.addCallback(complete)
        
        reactor.callLater(0, d.callback, None)
        return d