class Food(object):
def __init__(self):
self.db = Connection()["food"]["choices"]
def add(self, name):
name = str(name).lower()
self.db.update({'name':name}, {'name':name}, upsert=True)
def remove(self, name):
self.db.remove({'name':name})
def get_all(self):
retVal = []
try:
cur = self.db.find()
for i in cur:
retVal.append(i['name'])
except:
pass
return retVal
def choose(self):
try:
cur = self.db.find()
tmp = []
for i in cur:
tmp.append(i['name'])
index = random.randrange(len(tmp))
return tmp[index]
except:
return "unknown"
class Collective(object):
class Adaptive(object):
# See the XCS paper for details: Butz and Wilson, 2001
# http://citeseer.ist.psu.edu/old/700101.html
N = population_size = 1000
B = learning_rate = .1
a = accuracy_slope = .1
e0 = error_minimum = 1
v = power_parameter = -5
g = discount_factor = .8
OGA = ga_threshold = 40
X = crossover_prob = .75
u = mutation_prob = .01
Odel = experience_threshold = 20
d = fitness_threshold = .01
Osub = subsumption_threshold = 50
P = mask_probability = .3
p1 = initial_prediction = 1
e1 = initial_error = .1
F1 = initial_fitness = .01
pexplr = exploration_probability = .25
Omna = coverage_threshold = 4
# Reduced learning rate for the environmental error
Be = variance_learning_rate = .05
# Subsumption is probably not useful here.
doGASubsumption = False
doActionSetSubsumption = False
# Todo: Consider weak refs for this
singletons = {}
def __new__(cls, table):
try:
result = cls.singletons[table]
except KeyError:
result = object.__new__(cls)
cls.singletons[table] = result
result.init(table)
return result
def init(self, table):
# Not __init__, because that gets run too often.
self.values = self.Adaptive()
self.rules = self.retrieve(table)
self.timestamp = 0
def retrieve(self, table):
try:
from pymongo import Connection
except ImportError:
rules = []
else:
self.table = Connection().parang[table]
rules = [Classifier(row) for row in self.table.find()]
return rules
def save(self, rule):
# Called whenever a classifier is created or changed.
if self.table:
if rule.n > 0:
uid = self.table.save(rule.values())
rule._id = uid
elif rule._id:
self.table.remove(rule._id)
def generate(self, msg):
self.timestamp += 1
results = defaultdict(list)
for rule in self.rules:
output = rule.matches(msg)
if output is not None:
results[output].append(rule)
while sum(r.n for s in results for r in results[s]) < self.values.Omna:
rule = self.coverage(msg, results)
output = rule.matches(msg)
results[output].append(rule)
self.delete()
actions = dict((key, sum(r.p * r.F for r in results[key]) /
sum(r.F for r in results[key]))
for key in results
if results[key])
action = weighted_choice(actions)
action_set = results[action]
brigade = self.values.g * max(actions.values())
return action, action_set, brigade
def coverage(self, msg, actions):
r"Creates a new classifier to fit the under-represented message."
# Ignore only P% of the pattern bits.
mask = 0
for n in range(Classifier.bits):
if random() > self.values.P:
mask |= 1 << n
while True:
# Generate an action not in the match set.
# This also guarantees that the new classifier is unique.
action = randrange(1 << Classifier.bits)
if action not in actions:
break
values = {
"pattern": msg,
"pattern_mask": mask,
"output": action,
"output_mask": 0,
"prediction": self.values.p1,
"error": self.values.e1,
"fitness": self.values.F1,
"experience": 0,
"timestamp": self.timestamp,
"setsize": len(actions),
"numerosity": 1,
}
rule = Classifier(values)
self.rules.append(rule)
self.save(rule)
return rule
def update(self, action_set, bonus, msg):
# Update the action set
set_size = sum(rule.n for rule in action_set)
if not set_size:
# All classifiers have been deleted.
# Continuing would cause division by zero errors.
return
# Factor out the error due to environmental changes
ubar = min(bonus - rule.p for rule in action_set)
accuracy = 0
for rule in action_set:
rule.exp += 1
factor = max(1. / rule.exp, self.values.B)
# Reordering these updates may help for more complex problems.
rule.u += (ubar - rule.u) * self.values.Be
rule.p += (bonus - rule.p) * factor
err = abs(bonus - rule.p) - rule.u
if err < 0: err = self.values.e0
rule.e += (err - rule.e) * factor
rule.s += (set_size - rule.s) * factor
if rule.e < self.values.e0:
rule.k = 1
else:
rule.k = self.values.a * (rule.e / self.values.e0) ** self.values.v
accuracy += rule.k * rule.n
# Update the fitness separately, using the total accuracy.
for rule in action_set:
rule.F += (rule.k * rule.n / accuracy - rule.F) * self.values.B
for rule in action_set:
self.save(rule)
# Run the genetic algorithm every so often
avetime = sum(r.ts * r.n for r in action_set) / set_size
if self.timestamp - avetime > self.values.OGA:
self.genetic(action_set, msg)
def genetic(self, action_set, msg):
# Set timestamps for future use
for rule in action_set:
rule.ts = self.timestamp
# Choose two, based on their fitness values
fitness = dict((rule, rule.F) for rule in action_set)
first = weighted_choice(fitness).copy()
second = weighted_choice(fitness).copy()
if random() < self.values.X:
self.crossover(first, second)
self.mutate(first, msg)
self.mutate(second, msg)
self.insert(first)
self.insert(second)
self.delete()
def crossover(self, first, second):
x = randrange(Classifier.bits)
y = randrange(Classifier.bits)
if x > y:
x, y = y, x
mask = 0
for n in range(x, y + 1):
mask |= 1 << n
fp, fpm, fo, fom = first.unpack()
sp, spm, so, som = second.unpack()
# Swap the pattern, using the bitwise trick
fp ^= sp & mask
sp ^= fp & mask
fp ^= sp & mask
# Swap the pattern mask
fpm ^= spm & mask
spm ^= fpm & mask
fpm ^= spm & mask
first.pack(fp, fpm, fo, fom)
second.pack(sp, spm, so, som)
# Average out the performance measurements
first.p = second.p = (first.p + second.p) / 2
first.e = second.e = (first.e + second.e) / 2
first.F = second.F = (first.F + second.F) / 2
def mutate(self, rule, msg):
prob = self.values.u
pattern, pattern_mask, output, output_mask = rule.unpack()
for n in range(Classifier.bits):
bit = 1 << n
if random() < prob:
# Mutate only within the matching niche
pattern_mask ^= bit
if msg & bit:
pattern |= bit
else:
pattern &= ~bit
if random() < prob:
output ^= bit
if random() < prob:
output_mask ^= bit
# Save the new values
rule.pack(pattern, pattern_mask, output, output_mask)
# Temporarily decrease fitness
rule.F *= 0.1
def insert(self, rule):
for r in self.rules:
if r.chromosome == rule.chromosome:
r.n += rule.n
self.save(r)
break
else:
self.rules.append(rule)
self.save(rule)
def delete(self):
total = sum(rule.n for rule in self.rules)
excess = total - self.values.N
if excess < 1:
return []
fitness = sum(rule.F for rule in self.rules) / total
scores = dict((rule, self.unfitness(rule, fitness))
for rule in self.rules)
deleted = []
while excess > 0:
rule = weighted_choice(scores)
rule.n -= 1
if rule.n <= 0:
self.rules.remove(rule)
del scores[rule]
deleted.append(rule)
self.save(rule)
excess -= 1
return deleted
def unfitness(self, rule, average):
result = rule.n * rule.s
if rule.exp > self.values.Odel and rule.F < average * self.values.d * rule.n:
result *= average * rule.n / rule.F
return result
c = Connection()
db = c.burrito_db
proc_col = db.process_trace
gui_col = db.gui_trace
clipboard_col = db.clipboard_trace
xpad_col = db.apps.xpad
vim_col = db.apps.vim
bash_col = db.apps.bash
session_status_col = db.session_status
all_cols = [proc_col, gui_col, clipboard_col, xpad_col, vim_col, bash_col]
# First clear all entries matching session_tag:
for c in all_cols:
c.remove({"session_tag": session_tag})
session_status_col.remove({"_id": session_tag})
if options.delete_session:
print "Done deleting session named '%s'" % (session_tag, )
sys.exit(0)
# Create indices
# TODO: I don't know whether it's wasteful or dumb to KEEP creating
# these indices every time you start up the connection ...
proc_col.ensure_index('pid')
proc_col.ensure_index('exited')
proc_col.ensure_index('most_recent_event_timestamp')
# For time range searches! This multi-key index ensures fast
class Mongo(object):
def __init__(self, log, sw=None):
self.name = self.__class__.__name__
self.log = log
self.articles = Connection().aivb_db.articles \
if not sw else Connection().aivb_redux.dater
def __str__(self):
return """
'all': None,
'search': {k: v},
'empty': {k: 0},
'filled': {k: {'$gt': 0.5}},
'gtv': {k: {'$gt': v}},
'regex': {k: {'$regex': v}},
'exists': {k: {'$exists': True}},
'and_ex': {'$and': [{k: v}, {k2: {'$exists': True}}]},
'grt_ex': {'$and': [{k: {'$exists': True}}, {k2: {'$gt': v2}}]},
'grt_eq': {'$and': [{k: {'$exists': True}}, {k2: v2}]},
'p_range': {'$and': [{k: {'$gte': v}}, {k2: {'$lte': v2}}]},
'period': {'$and': [{k: v}, {k2: {'$gt': v2}}]},
'andand': {'$and': [{k: v}, {k2: v2}]}
"""
def load(self, n=None):
load = Loader(self.log)
data = load.fetch_data(n)
[[self.articles.insert(i) for i in x] for x in data]
self.log.mlog.info("Inserted %d Instances of articles." % n)
def search(self,
command,
key=None,
value=None,
s_key=None,
s_value=None,
t_key=None):
if not key:
res = [self.articles.find_one()]
else:
res = self.parse_search(command, key, value, s_key, s_value, t_key)
return res
def clear_all(self, v=None):
for art in self.articles.find():
if v:
print art
self.articles.remove(art)
def parse_search(self, c, k, v, k2, v2, k3):
op = {
'all': None,
'search': {
k: v
},
'empty': {
k: 0
},
'filled': {
k: {
'$gt': 0.5
}
},
'gtv': {
k: {
'$gt': v
}
},
'regex': {
k: {
'$regex': v
}
},
'exists': {
k: {
'$exists': True
}
},
'and_ex': {
'$and': [{
k: v
}, {
k2: {
'$exists': True
}
}]
},
'grt_ex': {
'$and': [{
k: {
'$exists': True
}
}, {
k2: {
'$gt': v2
}
}]
},
'grt_eq': {
'$and': [{
k: {
'$exists': True
}
}, {
k2: v2
}]
},
'p_range': {
'$and': [{
k: {
'$gte': v
}
}, {
k2: {
'$lte': v2
}
}]
},
'period': {
'$and': [{
k: v
}, {
k2: {
'$gt': v2
}
}]
},
'andand': {
'$and': [{
k: v
}, {
k2: v2
}]
}
}
if 'select' not in c:
return self.articles.find(op[c])
else:
if not k3:
return self.articles.find(op[c.split('_')[1]], {
'_id': k2,
v2: 1
})
else:
return self.articles.find(op[c.split('_')[1]], {
'_id': k2,
v2: 1,
k3: 1
})
def update(self, c, eid, k, v, k2=None):
op = {'one': {'$set': {k: v}}, 'two': {'$set': {k2: {'$set': {k: v}}}}}
self.articles.update({'_id': eid}, op[c], upsert=False, multi=False)
c = Connection()
db = c.burrito_db
proc_col = db.process_trace
gui_col = db.gui_trace
clipboard_col = db.clipboard_trace
xpad_col = db.apps.xpad
vim_col = db.apps.vim
bash_col = db.apps.bash
session_status_col = db.session_status
all_cols = [proc_col, gui_col, clipboard_col, xpad_col, vim_col, bash_col]
# First clear all entries matching session_tag:
for c in all_cols:
c.remove({"session_tag": session_tag})
session_status_col.remove({"_id": session_tag})
if options.delete_session:
print "Done deleting session named '%s'" % (session_tag,)
sys.exit(0)
# Create indices
# TODO: I don't know whether it's wasteful or dumb to KEEP creating
# these indices every time you start up the connection ...
proc_col.ensure_index('pid')
proc_col.ensure_index('exited')
proc_col.ensure_index('most_recent_event_timestamp')
