def merge(self, follower, m):
if follower != self.creator:
fdq = crane.mantisStore.load_one(
{
'name': self.name,
'creator': follower
}, {
'dq': True
}).get('dq', [])
fdq = FlexibleVector(generic=fdq)
else:
fdq = self.dq
rd = (fdq.norm() + EPS) / (self.panda.z.norm() + EPS)
if rd < self.eps:
logger.debug('Converged, no need to merge')
return False
else:
self.panda.z.add(fdq, 1.0 / (m + 1 / self.rho))
logger.debug('m = {0}'.format(m))
logger.debug('update z {0}'.format(self.panda.z))
logger.debug('relative difference of z {0}'.format(rd))
metricValue(self, 'rz', rd)
#self.panda.update_fields({self.panda.FCONSENSUS:self.panda.z.generic()})
if fdq is not self.dq:
del fdq
return True
def clone(self, userName, panda):
obj = super(Mantis, self).clone(userName)
obj.mu = FlexibleVector()
obj.dq = FlexibleVector()
obj.q = self.panda.z.clone()
obj.panda = panda
obj.solver = SVMDual(panda.weights, self.eps, self.rho, self.gamma,
self.maxNumIters, self.maxNumInstances)
obj.data = {}
return obj
def __restore__(self):
super(Mantis, self).__restore__()
self.solver = None
try:
self.mu = FlexibleVector(generic=self.mu)
self.q = FlexibleVector(generic=self.q)
self.dq = FlexibleVector(generic=self.dq)
self.data = {ObjectId(k) : v for k,v in self.data.iteritems()}
return True
except Exception as e:
logger.error('error {0}'.format(e.message))
logger.error('can not create a solver for {0}'.format(self.panda.name))
return False
def centralizedTest(isPersonalized):
global users
global testData
checkUserPartitionMapping()
mcl = pm.MongoClient('10.137.172.201:27017')
coll = mcl.DataSet['PMLExpression']
MONKModelPandaStore = mcl.MONKModel['PandaStore']
monkpa = MONKModelPandaStore.find_one({'creator': 'monk', 'name': pandaName}, {'_id':True, 'weights':True, 'z':True}, timeout=False)
z = FlexibleVector(generic=monkpa['z'])
resGTs = {}
for user in testData.keys():
if user == '':
continue
pa = MONKModelPandaStore.find_one({'creator': user, 'name': pandaName}, {'_id':True, 'weights':True, 'z':True}, timeout=False)
if pa == None:
continue
if isPersonalized == True:
wei = FlexibleVector(generic=pa['weights'])
else:
wei = z
resGT = []
for ent in coll.find({'_id': {'$in':testData[user]}}, {'_features':True, 'labels':True}, timeout=False):
fea = FlexibleVector(generic=ent['_features'])
if not len(ent['labels']) == 0:
resGT.append((float(wei.dot(fea)), 1.0))
else:
resGT.append((float(wei.dot(fea)), 0.0))
resGTs[user] = resGT
del wei
mcl.close()
return resGTs
def merge(self, follower, m):
if follower != self.creator:
fdq = crane.mantisStore.load_one({'name':self.name,
'creator':follower},
{'dq':True}).get('dq',[])
fdq = FlexibleVector(generic=fdq)
else:
fdq = self.dq
rd = (fdq.norm() + EPS) / (self.panda.z.norm() + EPS)
if rd < self.eps:
logger.debug('Converged, no need to merge')
return False
else:
self.panda.z.add(fdq, 1.0 / (m + 1 / self.rho))
logger.debug('m = {0}'.format(m))
logger.debug('update z {0}'.format(self.panda.z))
logger.debug('relative difference of z {0}'.format(rd))
metricValue(self, 'rz', rd)
#self.panda.update_fields({self.panda.FCONSENSUS:self.panda.z.generic()})
if fdq is not self.dq:
del fdq
return True
def __restore__(self):
super(Mantis, self).__restore__()
self.solver = None
try:
self.mu = FlexibleVector(generic=self.mu)
self.q = FlexibleVector(generic=self.q)
self.dq = FlexibleVector(generic=self.dq)
self.data = {ObjectId(k): v for k, v in self.data.iteritems()}
return True
except Exception as e:
logger.error('error {0}'.format(e.message))
logger.error('can not create a solver for {0}'.format(
self.panda.name))
return False
class LinearPanda(Panda):
FWEIGHTS = "weights"
FMANTIS = "mantis"
FCONSENSUS = "z"
FNUMFOLLOWERS = "m"
def __default__(self):
super(LinearPanda, self).__default__()
self.weights = []
self.z = []
self.mantis = None
self.m = 1
def __restore__(self):
super(LinearPanda, self).__restore__()
self.weights = FlexibleVector(generic=self.weights)
self.z = FlexibleVector(generic=self.z)
def generic(self):
result = super(LinearPanda, self).generic()
if self.mantis_loaded():
result[self.FMANTIS] = self.mantis.signature()
result[self.FWEIGHTS] = self.weights.generic()
result[self.FCONSENSUS] = self.z.generic()
return result
def clone(self, userName):
obj = super(LinearPanda, self).clone(userName)
obj.weights = self.weights.clone()
obj.z = obj.weights.clone()
obj.m = 1
self.load_mantis()
obj.mantis = self.mantis.clone(userName, obj)
return obj
def save(self):
super(LinearPanda, self).save()
if self.mantis_loaded():
self.mantis.save()
def delete(self):
result = super(LinearPanda, self).delete()
try:
result = result & self.mantis.delete()
except:
self.load_mantis()
result = result & self.mantis.delete()
return result
def has_mantis(self):
return True
def mantis_loaded(self):
return isinstance(self.mantis, Mantis)
def load_mantis(self):
if self.mantis_loaded():
return
if self.mantis is None:
self.mantis = {Mantis.MONK_TYPE: "Mantis", Mantis.NAME: self.name}
try:
self.mantis.setdefault(Mantis.CREATOR, self.creator)
self.mantis.setdefault(Mantis.FPANDA, self)
except:
logger.error("mantis should be a dict for loading")
logger.error("now is {0}".format(self.mantis))
return
self.mantis = crane.mantisStore.load_or_create(self.mantis, True)
self.mantis.initialize(self)
def add_data(self, entity, y, c):
try:
self.mantis.add_data(entity, y, c)
except:
self.load_mantis()
self.mantis.add_data(entity, y, c)
def increment(self):
self.m += 1
self.update_fields({self.FNUMFOLLOWERS: self.m})
def decrease(self):
self.m -= 1
self.update_fields({self.FNUMFOLLOWERS: self.m})
def add_features(self, uids):
self.weights.addKeys(uids)
self.z.addKeys(uids)
def pull_model(self):
genericW = self.store.load_one_in_fields(self, [self.FWEIGHTS, self.FCONSENSUS])
self.weights.update(genericW.get(self.FWEIGHTS, []))
self.z.update(genericW.get(self.FCONSENSUS, []))
def push_model(self):
self.update_fields({self.FWEIGHTS: self.weights.generic(), self.FCONSENSUS: self.z.generic()})
def train(self, leader):
try:
self.mantis.train(leader)
except:
self.load_mantis()
self.mantis.train(leader)
def checkout(self, leader):
try:
self.mantis.checkout(leader)
except:
self.load_mantis()
self.mantis.checkout(leader)
def commit(self):
self.push_model()
try:
self.mantis.commit()
except:
self.load_mantis()
self.mantis.commit()
def merge(self, follower):
try:
return self.mantis.merge(follower, self.m)
except:
self.load_mantis()
return self.mantis.merge(follower, self.m)
def predict(self, entity):
value = self.weights.dot(entity._features)
entity[self.uid] = sigmoid(value)
return entity[self.uid]
def reset(self):
self.weights.clear()
self.z.clear()
logger.debug("weights {0}".format(self.weights))
logger.debug("z {0}".format(self.z))
self.update_fields({self.FWEIGHTS: [], self.FCONSENSUS: []})
logger.debug("resetting mantis")
try:
self.mantis.reset()
except:
crane.mantisStore.update_in_fields(
{Mantis.NAME: self.name, Mantis.CREATOR: self.creator},
{Mantis.FDUALS: [], Mantis.FQ: [], Mantis.FDQ: []},
)
def reset_data(self):
logger.debug("resetting data in mantis")
try:
self.mantis.reset_data()
except:
crane.mantisStore.update_in_fields(
{Mantis.NAME: self.name, Mantis.CREATOR: self.creator}, {Mantis.FDATA: {}}
)
def set_mantis_parameter(self, para, value):
try:
self.mantis.set_mantis_parameter(para, value)
except:
self.load_mantis()
self.mantis.set_mantis_parameter(para, value)
def __restore__(self):
super(LinearPanda, self).__restore__()
self.weights = FlexibleVector(generic=self.weights)
self.z = FlexibleVector(generic=self.z)
def train(self, leader):
if not self.data:
logger.debug('no data, skip training')
return
logger.debug('gamma in mantis {0}'.format(self.gamma))
# check out z
if leader:
z = crane.pandaStore.load_one(
{
'name': self.panda.name,
'creator': leader
}, {
'z': True
}).get('z', [])
z = FlexibleVector(generic=z)
else:
z = self.panda.z
if z is None:
logger.debug('no consensus checked out')
return
#metricAbs(metricLog, self, '|z|', z)
#metricAbs(metricLog, self, '|q|', self.q)
metricRelAbs(self, 'z~q', self.q, z)
# update mu
self.dq.clear()
self.dq.add(self.mu, -1)
self.mu.add(self.q, 1)
self.mu.add(z, -1)
self.dq.add(self.mu, 1)
metricAbs(self, 'mu', self.mu)
#metricAbs(metricLog, self, '|dmu|', self.dq)
#metricValue(metricLog, self, 'sup(mu)', 2 * self.solver.num_instances * self.solver.maxxnorm() * z.norm())
# update w
self.solver.setModel0(z, self.mu)
#loss = self.solver.status()
#metricValue(metricLog, self, 'loss', loss)
#metricRelAbs(metricLog, self, '|q~w|', self.q, self.panda.weights)
#logger.debug('q = {0}'.format(self.q))
#logger.debug('w = {0}'.format(self.panda.weights))
self.solver.trainModel()
loss = self.solver.status()
metricValue(self, 'loss', loss)
metricValue(self, 'x', self.solver.maxxnorm())
# update q
r = self.rho / float(self.rho + self.gamma)
self.dq.add(self.q, -1)
self.q.clear()
self.q.add(z, r)
self.q.add(self.panda.weights, 1 - r)
self.q.add(self.mu, -r)
self.dq.add(self.q, 1)
if z is not self.panda.z:
del z
logger.debug('q = {0}'.format(self.q))
logger.debug('w = {0}'.format(self.panda.weights))
# measure convergence
#metricAbs(self, '|dq|', self.dq)
#metricAbs(self, '|q|', self.q)
metricRelAbs(self, 'q~w', self.q, self.panda.weights)
# commit changes
self.panda.update_fields(
{self.panda.FWEIGHTS: self.panda.weights.generic()})
self.commit()
class Mantis(base.MONKObject):
FEPS = 'eps'
FGAMMA = 'gamma'
FRHO = 'rho'
FPANDA = 'panda'
FDATA = 'data'
FDUALS = 'mu'
FQ = 'q'
FDQ = 'dq'
FMAX_NUM_ITERS = 'maxNumIters'
FMAX_NUM_INSTANCES = 'maxNumInstances'
store = crane.mantisStore
def __default__(self):
super(Mantis, self).__default__()
self.eps = 1e-4
self.gamma = 1
self.rho = 1
self.maxNumIters = 1000
self.maxNumInstances = 1000
self.panda = None
self.data = {}
self.mu = []
self.q = []
self.dq = []
def __restore__(self):
super(Mantis, self).__restore__()
self.solver = None
try:
self.mu = FlexibleVector(generic=self.mu)
self.q = FlexibleVector(generic=self.q)
self.dq = FlexibleVector(generic=self.dq)
self.data = {ObjectId(k): v for k, v in self.data.iteritems()}
return True
except Exception as e:
logger.error('error {0}'.format(e.message))
logger.error('can not create a solver for {0}'.format(
self.panda.name))
return False
def initialize(self, panda):
self.panda = panda
self.solver = SVMDual(self.panda.weights, self.eps, self.rho,
self.gamma, self.maxNumIters,
self.maxNumInstances)
ents = crane.entityStore.load_all_by_ids(self.data.keys())
for ent in ents:
index, y, c = self.data[ent._id]
self.solver.setData(ent._features, y, c, index)
self.solver.num_instances = len(ents)
keys = self.panda.weights.getKeys()
self.q.addKeys(keys)
self.dq.addKeys(keys)
def generic(self):
result = super(Mantis, self).generic()
# every mantis should have a panda
result[self.FPANDA] = self.panda._id
result[self.FDUALS] = self.mu.generic()
result[self.FQ] = self.q.generic()
result[self.FDQ] = self.dq.generic()
result[self.FDATA] = {str(k): v for k, v in self.data.iteritems()}
try:
del result['solver']
except Exception as e:
logger.warning('deleting solver failed {0}'.format(e.message))
return result
def clone(self, userName, panda):
obj = super(Mantis, self).clone(userName)
obj.mu = FlexibleVector()
obj.dq = FlexibleVector()
obj.q = self.panda.z.clone()
obj.panda = panda
obj.solver = SVMDual(panda.weights, self.eps, self.rho, self.gamma,
self.maxNumIters, self.maxNumInstances)
obj.data = {}
return obj
def train(self, leader):
if not self.data:
logger.debug('no data, skip training')
return
logger.debug('gamma in mantis {0}'.format(self.gamma))
# check out z
if leader:
z = crane.pandaStore.load_one(
{
'name': self.panda.name,
'creator': leader
}, {
'z': True
}).get('z', [])
z = FlexibleVector(generic=z)
else:
z = self.panda.z
if z is None:
logger.debug('no consensus checked out')
return
#metricAbs(metricLog, self, '|z|', z)
#metricAbs(metricLog, self, '|q|', self.q)
metricRelAbs(self, 'z~q', self.q, z)
# update mu
self.dq.clear()
self.dq.add(self.mu, -1)
self.mu.add(self.q, 1)
self.mu.add(z, -1)
self.dq.add(self.mu, 1)
metricAbs(self, 'mu', self.mu)
#metricAbs(metricLog, self, '|dmu|', self.dq)
#metricValue(metricLog, self, 'sup(mu)', 2 * self.solver.num_instances * self.solver.maxxnorm() * z.norm())
# update w
self.solver.setModel0(z, self.mu)
#loss = self.solver.status()
#metricValue(metricLog, self, 'loss', loss)
#metricRelAbs(metricLog, self, '|q~w|', self.q, self.panda.weights)
#logger.debug('q = {0}'.format(self.q))
#logger.debug('w = {0}'.format(self.panda.weights))
self.solver.trainModel()
loss = self.solver.status()
metricValue(self, 'loss', loss)
metricValue(self, 'x', self.solver.maxxnorm())
# update q
r = self.rho / float(self.rho + self.gamma)
self.dq.add(self.q, -1)
self.q.clear()
self.q.add(z, r)
self.q.add(self.panda.weights, 1 - r)
self.q.add(self.mu, -r)
self.dq.add(self.q, 1)
if z is not self.panda.z:
del z
logger.debug('q = {0}'.format(self.q))
logger.debug('w = {0}'.format(self.panda.weights))
# measure convergence
#metricAbs(self, '|dq|', self.dq)
#metricAbs(self, '|q|', self.q)
metricRelAbs(self, 'q~w', self.q, self.panda.weights)
# commit changes
self.panda.update_fields(
{self.panda.FWEIGHTS: self.panda.weights.generic()})
self.commit()
def checkout(self, leader):
pass
def merge(self, follower, m):
if follower != self.creator:
fdq = crane.mantisStore.load_one(
{
'name': self.name,
'creator': follower
}, {
'dq': True
}).get('dq', [])
fdq = FlexibleVector(generic=fdq)
else:
fdq = self.dq
rd = (fdq.norm() + EPS) / (self.panda.z.norm() + EPS)
if rd < self.eps:
logger.debug('Converged, no need to merge')
return False
else:
self.panda.z.add(fdq, 1.0 / (m + 1 / self.rho))
logger.debug('m = {0}'.format(m))
logger.debug('update z {0}'.format(self.panda.z))
logger.debug('relative difference of z {0}'.format(rd))
metricValue(self, 'rz', rd)
#self.panda.update_fields({self.panda.FCONSENSUS:self.panda.z.generic()})
if fdq is not self.dq:
del fdq
return True
def commit(self):
self.update_fields({
self.FDUALS: self.mu.generic(),
self.FQ: self.q.generic(),
self.FDQ: self.dq.generic()
})
def add_data(self, entity, y, c):
da = self.data
uuid = entity._id
if uuid in da:
ind = da[uuid][0]
elif self.solver.num_instances < self.maxNumInstances:
ind = self.solver.num_instances
self.solver.num_instances = ind + 1
else:
# random replacement policy
# TODO: should replace the most confident data
olduuid, (ind, oldy, oldc) = da.popitem()
self.solver.setData(entity._features, y, c, ind)
da[uuid] = (ind, y, c)
def reset(self):
self.mu.clear()
self.q.clear()
self.dq.clear()
logger.debug('mu {0}'.format(self.mu))
logger.debug('q {0}'.format(self.q))
logger.debug('dq {0}'.format(self.dq))
self.commit()
self.solver.initialize()
def reset_data(self):
self.data = {}
self.solver.num_instances = 0
logger.debug('data {0}'.format(self.data))
self.update_fields({self.FDATA: {}})
def set_mantis_parameter(self, para, value):
if (para == 'gamma'):
self.gamma = value
self.solver.setGamma(value)
logger.debug('gamma is {0}'.format(self.gamma))
logger.debug('gamma of solver is {0}'.format(self.solver.gamma))
self.update_fields({self.FGAMMA: self.gamma})
class LinearPanda(Panda):
FWEIGHTS = 'weights'
FMANTIS = 'mantis'
FCONSENSUS = 'z'
FNUMFOLLOWERS = 'm'
def __default__(self):
super(LinearPanda, self).__default__()
self.weights = []
self.z = []
self.mantis = None
self.m = 1
def __restore__(self):
super(LinearPanda, self).__restore__()
self.weights = FlexibleVector(generic=self.weights)
self.z = FlexibleVector(generic=self.z)
def generic(self):
result = super(LinearPanda, self).generic()
if self.mantis_loaded():
result[self.FMANTIS] = self.mantis.signature()
result[self.FWEIGHTS] = self.weights.generic()
result[self.FCONSENSUS] = self.z.generic()
return result
def clone(self, userName):
obj = super(LinearPanda, self).clone(userName)
obj.weights = self.weights.clone()
obj.z = obj.weights.clone()
obj.m = 1
self.load_mantis()
obj.mantis = self.mantis.clone(userName, obj)
return obj
def save(self):
super(LinearPanda, self).save()
if self.mantis_loaded():
self.mantis.save()
def delete(self):
result = super(LinearPanda, self).delete()
try:
result = result & self.mantis.delete()
except:
self.load_mantis()
result = result & self.mantis.delete()
return result
def has_mantis(self):
return True
def mantis_loaded(self):
return isinstance(self.mantis, Mantis)
def load_mantis(self):
if self.mantis_loaded():
return
if self.mantis is None:
self.mantis = {Mantis.MONK_TYPE: 'Mantis', Mantis.NAME: self.name}
try:
self.mantis.setdefault(Mantis.CREATOR, self.creator)
self.mantis.setdefault(Mantis.FPANDA, self)
except:
logger.error('mantis should be a dict for loading')
logger.error('now is {0}'.format(self.mantis))
return
self.mantis = crane.mantisStore.load_or_create(self.mantis, True)
self.mantis.initialize(self)
def add_data(self, entity, y, c):
try:
self.mantis.add_data(entity, y, c)
except:
self.load_mantis()
self.mantis.add_data(entity, y, c)
def increment(self):
self.m += 1
self.update_fields({self.FNUMFOLLOWERS: self.m})
def decrease(self):
self.m -= 1
self.update_fields({self.FNUMFOLLOWERS: self.m})
def add_features(self, uids):
self.weights.addKeys(uids)
self.z.addKeys(uids)
def pull_model(self):
genericW = self.store.load_one_in_fields(
self, [self.FWEIGHTS, self.FCONSENSUS])
self.weights.update(genericW.get(self.FWEIGHTS, []))
self.z.update(genericW.get(self.FCONSENSUS, []))
def push_model(self):
self.update_fields({
self.FWEIGHTS: self.weights.generic(),
self.FCONSENSUS: self.z.generic()
})
def train(self, leader):
try:
self.mantis.train(leader)
except:
self.load_mantis()
self.mantis.train(leader)
def checkout(self, leader):
try:
self.mantis.checkout(leader)
except:
self.load_mantis()
self.mantis.checkout(leader)
def commit(self):
self.push_model()
try:
self.mantis.commit()
except:
self.load_mantis()
self.mantis.commit()
def merge(self, follower):
try:
return self.mantis.merge(follower, self.m)
except:
self.load_mantis()
return self.mantis.merge(follower, self.m)
def predict(self, entity):
value = self.weights.dot(entity._features)
entity[self.uid] = sigmoid(value)
return entity[self.uid]
def reset(self):
self.weights.clear()
self.z.clear()
logger.debug('weights {0}'.format(self.weights))
logger.debug('z {0}'.format(self.z))
self.update_fields({self.FWEIGHTS: [], self.FCONSENSUS: []})
logger.debug('resetting mantis')
try:
self.mantis.reset()
except:
crane.mantisStore.update_in_fields(
{
Mantis.NAME: self.name,
Mantis.CREATOR: self.creator
}, {
Mantis.FDUALS: [],
Mantis.FQ: [],
Mantis.FDQ: []
})
def reset_data(self):
logger.debug('resetting data in mantis')
try:
self.mantis.reset_data()
except:
crane.mantisStore.update_in_fields(
{
Mantis.NAME: self.name,
Mantis.CREATOR: self.creator
}, {Mantis.FDATA: {}})
def set_mantis_parameter(self, para, value):
try:
self.mantis.set_mantis_parameter(para, value)
except:
self.load_mantis()
self.mantis.set_mantis_parameter(para, value)
def __restore__(self):
super(LinearPanda, self).__restore__()
self.weights = FlexibleVector(generic=self.weights)
self.z = FlexibleVector(generic=self.z)
class Mantis(base.MONKObject):
FEPS = 'eps'
FGAMMA = 'gamma'
FRHO = 'rho'
FPANDA = 'panda'
FDATA = 'data'
FDUALS = 'mu'
FQ = 'q'
FDQ = 'dq'
FMAX_NUM_ITERS = 'maxNumIters'
FMAX_NUM_INSTANCES = 'maxNumInstances'
store = crane.mantisStore
def __default__(self):
super(Mantis, self).__default__()
self.eps = 1e-4
self.gamma = 1
self.rho = 1
self.maxNumIters = 1000
self.maxNumInstances = 1000
self.panda = None
self.data = {}
self.mu = []
self.q = []
self.dq = []
def __restore__(self):
super(Mantis, self).__restore__()
self.solver = None
try:
self.mu = FlexibleVector(generic=self.mu)
self.q = FlexibleVector(generic=self.q)
self.dq = FlexibleVector(generic=self.dq)
self.data = {ObjectId(k) : v for k,v in self.data.iteritems()}
return True
except Exception as e:
logger.error('error {0}'.format(e.message))
logger.error('can not create a solver for {0}'.format(self.panda.name))
return False
def initialize(self, panda):
self.panda = panda
self.solver = SVMDual(self.panda.weights, self.eps, self.rho, self.gamma,
self.maxNumIters, self.maxNumInstances)
ents = crane.entityStore.load_all_by_ids(self.data.keys())
for ent in ents:
index, y, c = self.data[ent._id]
self.solver.setData(ent._features, y, c, index)
self.solver.num_instances = len(ents)
keys = self.panda.weights.getKeys()
self.q.addKeys(keys)
self.dq.addKeys(keys)
def generic(self):
result = super(Mantis, self).generic()
# every mantis should have a panda
result[self.FPANDA] = self.panda._id
result[self.FDUALS] = self.mu.generic()
result[self.FQ] = self.q.generic()
result[self.FDQ] = self.dq.generic()
result[self.FDATA] = {str(k) : v for k,v in self.data.iteritems()}
try:
del result['solver']
except Exception as e:
logger.warning('deleting solver failed {0}'.format(e.message))
return result
def clone(self, userName, panda):
obj = super(Mantis, self).clone(userName)
obj.mu = FlexibleVector()
obj.dq = FlexibleVector()
obj.q = self.panda.z.clone()
obj.panda = panda
obj.solver = SVMDual(panda.weights, self.eps, self.rho, self.gamma,
self.maxNumIters, self.maxNumInstances)
obj.data = {}
return obj
def train(self, leader):
if not self.data:
logger.debug('no data, skip training')
return
logger.debug('gamma in mantis {0}'.format(self.gamma))
# check out z
if leader:
z = crane.pandaStore.load_one({'name':self.panda.name,
'creator':leader},
{'z':True}).get('z',[])
z = FlexibleVector(generic=z)
else:
z = self.panda.z
if z is None:
logger.debug('no consensus checked out')
return
#metricAbs(metricLog, self, '|z|', z)
#metricAbs(metricLog, self, '|q|', self.q)
metricRelAbs(self, 'z~q', self.q, z)
# update mu
self.dq.clear()
self.dq.add(self.mu, -1)
self.mu.add(self.q, 1)
self.mu.add(z, -1)
self.dq.add(self.mu, 1)
metricAbs(self, 'mu', self.mu)
#metricAbs(metricLog, self, '|dmu|', self.dq)
#metricValue(metricLog, self, 'sup(mu)', 2 * self.solver.num_instances * self.solver.maxxnorm() * z.norm())
# update w
self.solver.setModel0(z, self.mu)
#loss = self.solver.status()
#metricValue(metricLog, self, 'loss', loss)
#metricRelAbs(metricLog, self, '|q~w|', self.q, self.panda.weights)
#logger.debug('q = {0}'.format(self.q))
#logger.debug('w = {0}'.format(self.panda.weights))
self.solver.trainModel()
loss = self.solver.status()
metricValue(self, 'loss', loss)
metricValue(self, 'x', self.solver.maxxnorm())
# update q
r = self.rho / float(self.rho + self.gamma)
self.dq.add(self.q, -1)
self.q.clear()
self.q.add(z, r)
self.q.add(self.panda.weights, 1 - r)
self.q.add(self.mu, -r)
self.dq.add(self.q, 1)
if z is not self.panda.z:
del z
logger.debug('q = {0}'.format(self.q))
logger.debug('w = {0}'.format(self.panda.weights))
# measure convergence
#metricAbs(self, '|dq|', self.dq)
#metricAbs(self, '|q|', self.q)
metricRelAbs(self, 'q~w', self.q, self.panda.weights)
# commit changes
self.panda.update_fields({self.panda.FWEIGHTS:self.panda.weights.generic()})
self.commit()
def checkout(self, leader):
pass
def merge(self, follower, m):
if follower != self.creator:
fdq = crane.mantisStore.load_one({'name':self.name,
'creator':follower},
{'dq':True}).get('dq',[])
fdq = FlexibleVector(generic=fdq)
else:
fdq = self.dq
rd = (fdq.norm() + EPS) / (self.panda.z.norm() + EPS)
if rd < self.eps:
logger.debug('Converged, no need to merge')
return False
else:
self.panda.z.add(fdq, 1.0 / (m + 1 / self.rho))
logger.debug('m = {0}'.format(m))
logger.debug('update z {0}'.format(self.panda.z))
logger.debug('relative difference of z {0}'.format(rd))
metricValue(self, 'rz', rd)
#self.panda.update_fields({self.panda.FCONSENSUS:self.panda.z.generic()})
if fdq is not self.dq:
del fdq
return True
def commit(self):
self.update_fields({self.FDUALS : self.mu.generic(),
self.FQ : self.q.generic(),
self.FDQ : self.dq.generic()})
def add_data(self, entity, y, c):
da = self.data
uuid = entity._id
if uuid in da:
ind = da[uuid][0]
elif self.solver.num_instances < self.maxNumInstances:
ind = self.solver.num_instances
self.solver.num_instances = ind + 1
else:
# random replacement policy
# TODO: should replace the most confident data
olduuid, (ind, oldy, oldc) = da.popitem()
self.solver.setData(entity._features, y, c, ind)
da[uuid] = (ind, y, c)
def reset(self):
self.mu.clear()
self.q.clear()
self.dq.clear()
logger.debug('mu {0}'.format(self.mu))
logger.debug('q {0}'.format(self.q))
logger.debug('dq {0}'.format(self.dq))
self.commit()
self.solver.initialize()
def reset_data(self):
self.data = {}
self.solver.num_instances = 0
logger.debug('data {0}'.format(self.data))
self.update_fields({self.FDATA : {}})
def set_mantis_parameter(self, para, value):
if (para == 'gamma'):
self.gamma = value
self.solver.setGamma(value)
logger.debug('gamma is {0}'.format(self.gamma))
logger.debug('gamma of solver is {0}'.format(self.solver.gamma))
self.update_fields({self.FGAMMA : self.gamma})