コード例 #1
0
def centralizedTest(isPersonalized):
    global users
    global testData
    checkUserPartitionMapping()
    
    mcl = pm.MongoClient('10.137.172.201:27017')
    coll = mcl.DataSet['PMLExpression']
    MONKModelPandaStore = mcl.MONKModel['PandaStore']
    monkpa = MONKModelPandaStore.find_one({'creator': 'monk', 'name': pandaName}, {'_id':True, 'weights':True, 'z':True}, timeout=False)
    z = FlexibleVector(generic=monkpa['z'])        
    resGTs = {}
    for user in testData.keys():
        if user == '':
            continue
        pa = MONKModelPandaStore.find_one({'creator': user, 'name': pandaName}, {'_id':True, 'weights':True, 'z':True}, timeout=False)
        if pa == None:
            continue
        if isPersonalized == True:
            wei = FlexibleVector(generic=pa['weights'])
        else:            
            wei = z
        resGT = []
        for ent in coll.find({'_id': {'$in':testData[user]}}, {'_features':True, 'labels':True}, timeout=False):     
            fea = FlexibleVector(generic=ent['_features'])   
            if not len(ent['labels']) == 0:
                resGT.append((float(wei.dot(fea)), 1.0))
            else:
                resGT.append((float(wei.dot(fea)), 0.0))
        resGTs[user] = resGT
        del wei
    mcl.close()
    return resGTs              
コード例 #2
0
ファイル: mantis.py プロジェクト: TravelGene-ios/travelgene
 def __restore__(self):
     super(Mantis, self).__restore__()
     self.solver = None
     
     try:
         self.mu = FlexibleVector(generic=self.mu)
         self.q  = FlexibleVector(generic=self.q)
         self.dq = FlexibleVector(generic=self.dq)
         self.data = {ObjectId(k) : v for k,v in self.data.iteritems()}
         return True
     except Exception as e:
         logger.error('error {0}'.format(e.message))
         logger.error('can not create a solver for {0}'.format(self.panda.name))
         return False
コード例 #3
0
ファイル: mantis.py プロジェクト: TravelGene-ios/travelgene
    def merge(self, follower, m):
        if follower != self.creator:
            fdq = crane.mantisStore.load_one({'name':self.name,
                                              'creator':follower},
                                             {'dq':True}).get('dq',[])
            fdq = FlexibleVector(generic=fdq)
        else:
            fdq = self.dq

        rd = (fdq.norm() + EPS) / (self.panda.z.norm() + EPS)
        if rd < self.eps:
            logger.debug('Converged, no need to merge')
            return False
        else:
            self.panda.z.add(fdq, 1.0 / (m + 1 / self.rho))
            logger.debug('m = {0}'.format(m))
            logger.debug('update z {0}'.format(self.panda.z))
            logger.debug('relative difference of z {0}'.format(rd))
            metricValue(self, 'rz', rd)
            #self.panda.update_fields({self.panda.FCONSENSUS:self.panda.z.generic()})
        
        if fdq is not self.dq:
            del fdq
        return True
コード例 #4
0
ファイル: panda.py プロジェクト: TravelGene-ios/travelgene
class LinearPanda(Panda):
    FWEIGHTS = "weights"
    FMANTIS = "mantis"
    FCONSENSUS = "z"
    FNUMFOLLOWERS = "m"

    def __default__(self):
        super(LinearPanda, self).__default__()
        self.weights = []
        self.z = []
        self.mantis = None
        self.m = 1

    def __restore__(self):
        super(LinearPanda, self).__restore__()
        self.weights = FlexibleVector(generic=self.weights)
        self.z = FlexibleVector(generic=self.z)

    def generic(self):
        result = super(LinearPanda, self).generic()
        if self.mantis_loaded():
            result[self.FMANTIS] = self.mantis.signature()
        result[self.FWEIGHTS] = self.weights.generic()
        result[self.FCONSENSUS] = self.z.generic()
        return result

    def clone(self, userName):
        obj = super(LinearPanda, self).clone(userName)
        obj.weights = self.weights.clone()
        obj.z = obj.weights.clone()
        obj.m = 1
        self.load_mantis()
        obj.mantis = self.mantis.clone(userName, obj)
        return obj

    def save(self):
        super(LinearPanda, self).save()
        if self.mantis_loaded():
            self.mantis.save()

    def delete(self):
        result = super(LinearPanda, self).delete()
        try:
            result = result & self.mantis.delete()
        except:
            self.load_mantis()
            result = result & self.mantis.delete()
        return result

    def has_mantis(self):
        return True

    def mantis_loaded(self):
        return isinstance(self.mantis, Mantis)

    def load_mantis(self):
        if self.mantis_loaded():
            return

        if self.mantis is None:
            self.mantis = {Mantis.MONK_TYPE: "Mantis", Mantis.NAME: self.name}

        try:
            self.mantis.setdefault(Mantis.CREATOR, self.creator)
            self.mantis.setdefault(Mantis.FPANDA, self)
        except:
            logger.error("mantis should be a dict for loading")
            logger.error("now is {0}".format(self.mantis))
            return

        self.mantis = crane.mantisStore.load_or_create(self.mantis, True)
        self.mantis.initialize(self)

    def add_data(self, entity, y, c):
        try:
            self.mantis.add_data(entity, y, c)
        except:
            self.load_mantis()
            self.mantis.add_data(entity, y, c)

    def increment(self):
        self.m += 1
        self.update_fields({self.FNUMFOLLOWERS: self.m})

    def decrease(self):
        self.m -= 1
        self.update_fields({self.FNUMFOLLOWERS: self.m})

    def add_features(self, uids):
        self.weights.addKeys(uids)
        self.z.addKeys(uids)

    def pull_model(self):
        genericW = self.store.load_one_in_fields(self, [self.FWEIGHTS, self.FCONSENSUS])
        self.weights.update(genericW.get(self.FWEIGHTS, []))
        self.z.update(genericW.get(self.FCONSENSUS, []))

    def push_model(self):
        self.update_fields({self.FWEIGHTS: self.weights.generic(), self.FCONSENSUS: self.z.generic()})

    def train(self, leader):
        try:
            self.mantis.train(leader)
        except:
            self.load_mantis()
            self.mantis.train(leader)

    def checkout(self, leader):
        try:
            self.mantis.checkout(leader)
        except:
            self.load_mantis()
            self.mantis.checkout(leader)

    def commit(self):
        self.push_model()
        try:
            self.mantis.commit()
        except:
            self.load_mantis()
            self.mantis.commit()

    def merge(self, follower):
        try:
            return self.mantis.merge(follower, self.m)
        except:
            self.load_mantis()
            return self.mantis.merge(follower, self.m)

    def predict(self, entity):
        value = self.weights.dot(entity._features)
        entity[self.uid] = sigmoid(value)
        return entity[self.uid]

    def reset(self):
        self.weights.clear()
        self.z.clear()
        logger.debug("weights {0}".format(self.weights))
        logger.debug("z {0}".format(self.z))
        self.update_fields({self.FWEIGHTS: [], self.FCONSENSUS: []})
        logger.debug("resetting mantis")
        try:
            self.mantis.reset()
        except:
            crane.mantisStore.update_in_fields(
                {Mantis.NAME: self.name, Mantis.CREATOR: self.creator},
                {Mantis.FDUALS: [], Mantis.FQ: [], Mantis.FDQ: []},
            )

    def reset_data(self):
        logger.debug("resetting data in mantis")
        try:
            self.mantis.reset_data()
        except:
            crane.mantisStore.update_in_fields(
                {Mantis.NAME: self.name, Mantis.CREATOR: self.creator}, {Mantis.FDATA: {}}
            )

    def set_mantis_parameter(self, para, value):
        try:
            self.mantis.set_mantis_parameter(para, value)
        except:
            self.load_mantis()
            self.mantis.set_mantis_parameter(para, value)
コード例 #5
0
ファイル: panda.py プロジェクト: TravelGene-ios/travelgene
 def __restore__(self):
     super(LinearPanda, self).__restore__()
     self.weights = FlexibleVector(generic=self.weights)
     self.z = FlexibleVector(generic=self.z)
コード例 #6
0
ファイル: mantis.py プロジェクト: TravelGene-ios/travelgene
class Mantis(base.MONKObject):
    FEPS   = 'eps'
    FGAMMA = 'gamma'
    FRHO   = 'rho'
    FPANDA = 'panda'
    FDATA  = 'data'
    FDUALS = 'mu'
    FQ     = 'q'
    FDQ    = 'dq'
    FMAX_NUM_ITERS = 'maxNumIters'
    FMAX_NUM_INSTANCES = 'maxNumInstances'
    store = crane.mantisStore
    
    def __default__(self):
        super(Mantis, self).__default__()
        self.eps = 1e-4
        self.gamma = 1
        self.rho = 1
        self.maxNumIters = 1000
        self.maxNumInstances = 1000
        self.panda = None
        self.data = {}
        self.mu = []
        self.q  = []
        self.dq = []
        
    def __restore__(self):
        super(Mantis, self).__restore__()
        self.solver = None
        
        try:
            self.mu = FlexibleVector(generic=self.mu)
            self.q  = FlexibleVector(generic=self.q)
            self.dq = FlexibleVector(generic=self.dq)
            self.data = {ObjectId(k) : v for k,v in self.data.iteritems()}
            return True
        except Exception as e:
            logger.error('error {0}'.format(e.message))
            logger.error('can not create a solver for {0}'.format(self.panda.name))
            return False
            
    def initialize(self, panda):
        self.panda = panda
        self.solver = SVMDual(self.panda.weights, self.eps, self.rho, self.gamma,
                              self.maxNumIters, self.maxNumInstances)
        ents = crane.entityStore.load_all_by_ids(self.data.keys())
        for ent in ents:
            index, y, c = self.data[ent._id]
            self.solver.setData(ent._features, y, c, index)
        self.solver.num_instances = len(ents)
        keys = self.panda.weights.getKeys()
        self.q.addKeys(keys)
        self.dq.addKeys(keys)
        
    def generic(self):
        result = super(Mantis, self).generic()
        # every mantis should have a panda
        result[self.FPANDA] = self.panda._id
        result[self.FDUALS] = self.mu.generic()
        result[self.FQ]     = self.q.generic()
        result[self.FDQ]    = self.dq.generic()
        result[self.FDATA]  = {str(k) : v for k,v in self.data.iteritems()}
        try:
            del result['solver']
        except Exception as e:
            logger.warning('deleting solver failed {0}'.format(e.message))
        return result

    def clone(self, userName, panda):
        obj = super(Mantis, self).clone(userName)
        obj.mu = FlexibleVector()
        obj.dq = FlexibleVector()
        obj.q  = self.panda.z.clone()
        obj.panda = panda
        obj.solver = SVMDual(panda.weights, self.eps, self.rho, self.gamma,
                             self.maxNumIters, self.maxNumInstances)
        obj.data = {}
        return obj
    
    def train(self, leader):
        if not self.data:
            logger.debug('no data, skip training')
            return
            
        logger.debug('gamma in mantis {0}'.format(self.gamma))

        # check out z
        if leader:
            z = crane.pandaStore.load_one({'name':self.panda.name,
                                           'creator':leader},
                                          {'z':True}).get('z',[])
            z = FlexibleVector(generic=z)
        else:
            z = self.panda.z
        
        if z is None:
            logger.debug('no consensus checked out')
            return
            
        #metricAbs(metricLog, self, '|z|', z)
        #metricAbs(metricLog, self, '|q|', self.q)
        metricRelAbs(self, 'z~q', self.q, z)
        
        # update mu
        self.dq.clear()
        self.dq.add(self.mu, -1)
        self.mu.add(self.q, 1)
        self.mu.add(z, -1)
        self.dq.add(self.mu, 1)
        metricAbs(self, 'mu', self.mu)
        #metricAbs(metricLog, self, '|dmu|', self.dq)
        #metricValue(metricLog, self, 'sup(mu)', 2 * self.solver.num_instances * self.solver.maxxnorm() * z.norm())
        
        # update w
        self.solver.setModel0(z, self.mu)
        #loss = self.solver.status()
        #metricValue(metricLog, self, 'loss', loss)
        #metricRelAbs(metricLog, self, '|q~w|', self.q, self.panda.weights)
        #logger.debug('q = {0}'.format(self.q))
        #logger.debug('w = {0}'.format(self.panda.weights))
        self.solver.trainModel()
        loss = self.solver.status()
        metricValue(self, 'loss', loss)
        metricValue(self, 'x', self.solver.maxxnorm())
        
        # update q
        r = self.rho / float(self.rho + self.gamma)
        self.dq.add(self.q, -1)
        self.q.clear()
        self.q.add(z, r)
        self.q.add(self.panda.weights, 1 - r)
        self.q.add(self.mu, -r)
        self.dq.add(self.q, 1)
        
        if z is not self.panda.z:
            del z
            
        logger.debug('q = {0}'.format(self.q))
        logger.debug('w = {0}'.format(self.panda.weights))
        
        # measure convergence
        #metricAbs(self, '|dq|', self.dq)
        #metricAbs(self, '|q|', self.q)
        metricRelAbs(self, 'q~w', self.q, self.panda.weights)

        # commit changes
        self.panda.update_fields({self.panda.FWEIGHTS:self.panda.weights.generic()})
        self.commit()
    
    def checkout(self, leader):
        pass

    def merge(self, follower, m):
        if follower != self.creator:
            fdq = crane.mantisStore.load_one({'name':self.name,
                                              'creator':follower},
                                             {'dq':True}).get('dq',[])
            fdq = FlexibleVector(generic=fdq)
        else:
            fdq = self.dq

        rd = (fdq.norm() + EPS) / (self.panda.z.norm() + EPS)
        if rd < self.eps:
            logger.debug('Converged, no need to merge')
            return False
        else:
            self.panda.z.add(fdq, 1.0 / (m + 1 / self.rho))
            logger.debug('m = {0}'.format(m))
            logger.debug('update z {0}'.format(self.panda.z))
            logger.debug('relative difference of z {0}'.format(rd))
            metricValue(self, 'rz', rd)
            #self.panda.update_fields({self.panda.FCONSENSUS:self.panda.z.generic()})
        
        if fdq is not self.dq:
            del fdq
        return True
        
    def commit(self):
        self.update_fields({self.FDUALS : self.mu.generic(),
                            self.FQ     : self.q.generic(),
                            self.FDQ    : self.dq.generic()})
    
    def add_data(self, entity, y, c):
        da = self.data
        uuid = entity._id
        if uuid in da:
            ind = da[uuid][0] 
        elif self.solver.num_instances < self.maxNumInstances:
            ind = self.solver.num_instances
            self.solver.num_instances = ind + 1
        else:
            # random replacement policy
            # TODO: should replace the most confident data
            olduuid, (ind, oldy, oldc)  = da.popitem()
        self.solver.setData(entity._features, y, c, ind)
        da[uuid] = (ind, y, c)
        
    def reset(self):
        self.mu.clear()
        self.q.clear()
        self.dq.clear()
        logger.debug('mu {0}'.format(self.mu))
        logger.debug('q  {0}'.format(self.q))
        logger.debug('dq {0}'.format(self.dq))
        self.commit()
        self.solver.initialize()
        
    def reset_data(self):
        self.data = {}
        self.solver.num_instances = 0
        logger.debug('data {0}'.format(self.data))
        self.update_fields({self.FDATA : {}})
        
    def set_mantis_parameter(self, para, value):
        if (para == 'gamma'):
            self.gamma = value
            self.solver.setGamma(value)
            logger.debug('gamma is {0}'.format(self.gamma))
            logger.debug('gamma of solver is {0}'.format(self.solver.gamma))
            self.update_fields({self.FGAMMA : self.gamma})