def _parse_bdd(self, root):
if root.IsConstant():
return np.array([-1,-1,-1, root.IsComplement()]).reshape((1,4))
to_visit = [root]
visited = set([hash(root)])
visited_nodes = [root]
while to_visit:
curr_node = to_visit.pop()
high,low = curr_node.T(), curr_node.E()
if hash(low) not in visited:
if not low.IsConstant():
to_visit.append(low)
visited.add(hash(low))
visited_nodes.append(low)
if hash(high) not in visited:
if not high.IsConstant():
to_visit.append(high)
visited.add(hash(high))
visited_nodes.append(high)
sorted_nodes = sorted(visited_nodes, key= lambda x: x.NodeReadIndex(),
reverse=True)
logd('No. nodes: {}'.format(len(sorted_nodes)))
h_nodes = map(hash, sorted_nodes)
bdd = np.zeros((len(sorted_nodes), 4), dtype=np.int)
for i,node in enumerate(sorted_nodes):
if node.IsConstant():
bdd[i] = [-1, -1, -1, node.IsComplement()]
else:
high, low = node.T(), node.E()
high_idx, low_idx = map(lambda x: h_nodes.index(hash(x)),
[high, low])
bdd[i] = [node.NodeReadIndex(), high_idx,
low_idx, node.IsComplement()]
#logd(bdd)
return bdd
def compile_pycudd2(self, f_str, n_vars):
# initialize PyCUDD
mgr = pycudd.DdManager()
mgr.SetDefault()
# add variables
#pattern = '\d+'
#to_var = lambda match : 'v'+str(int(match.group(0))-1)
#to_exec = re.sub(pattern, to_var, f_str)
#logd(to_exec)
for i in range(n_vars):
exec 'v{} = mgr.IthVar({})'.format(i,i) in locals()
logd('ok1')
exec 'f = {}'.format(f_str) in locals()
logd('ok2')
# Debug BDD
#if n_vars>10:
f.DumpDot()
#logd(f.T().T().NodeReadIndex())
#logd(f.E().E().NodeReadIndex())
#root = f
# OLD compute node parents
#parents, nodes, h_nodes, idx_nodes = self._compute_node_parents(root)
#return mgr, root, parents, nodes, h_nodes, idx_nodes
return self._parse_bdd(f)
def ugs_samples(self, samples):
thetas, alphas, logbetas = zip(*samples)
self.last_theta = thetas[-1]
self.theta_avg = PBInference.pb_avg(thetas)
#logd('theta avg')
#for theta_it in self.theta_avg:
# logd(theta_it)
self.alpha_avg = PBInference.pb_avg(alphas)
logd(np.array(logbetas).shape)
#logd(betas)
#logd(len(betas)) # 100
#logd(len(betas[0])) # 361
#logd(float(betas[0][0]))
#logd(np.sum(np.average(logbetas, axis=0)))
self.total_ll = 0
duds = 0
for pb_plate in self.pb_model.plates:
if pb_plate.bdd is None:
duds += 1
self.total_ll += np.log(10**-6)
self.total_ll += np.average(np.sum(logbetas, axis=1))
logd('total_ll')
logd(self.total_ll)
logd('duds')
logd(duds)
def GetTypeOfCard(uid):
logd("GetTypeOfCard. uid: %s" %uid)
curCard = database.GetCard(uid)
if curCard != None:
return curCard['Type']
else:
return CardTypeUnknown
def test_rim1(self):
logd('### RIM1 ###')
cmd_str = 'peircebayes ./rim/rim_example1.pb -d'
PBCoreTests.call_pb(cmd_str)
with open('/tmp/peircebayes/aprob/plates/out1.plate', 'r') as fin:
out1 = fin.read()
with open('rim/rim1_test.plate', 'r') as fin:
test_out = fin.read()
self.assertEqual(out1, test_out)
def parse_and_compile(debug, tmp_dir, parallel_kc):
"""
Parse files produced by :func:`logical_inference` and compile them to a :class:`formula_gen.PBModel`.
:rtype: :class:`formula_gen.PBModel`
"""
option_args = {
"probs_file": os.path.join(tmp_dir, "aprob", "out.probs"),
"dir_file": os.path.join(tmp_dir, "aprob", "plates"),
}
start = time.time()
model = PBModel(option_args)
logd("bla")
# logd(model.plates)
logd("formula_gen takes: {}".format(time.time() - start))
logd("parallel_kc: {}".format(parallel_kc))
start = time.time()
if parallel_kc:
compile_k_parallel(model, debug, tmp_dir)
else:
compile_k(model, debug, tmp_dir)
logd("kc takes: {}".format(time.time() - start))
## prune empty bdds
##model.plates = [plate for plate in model.plates if not(plate.bdd is None)]
# logd(model)
return model
def sample_node_4(self, node_no_bft, curr_node, h_curr, pb_plate, beta, plate_idx, h_nodes):
high, low = curr_node.T(), curr_node.E()
h_high, h_low = map(myhash, [high, low])
i_high = pb_plate.bdd.h_nodes.index(h_high)
i_low = pb_plate.bdd.h_nodes.index(h_low)
i_curr = pb_plate.bdd.h_nodes.index(h_curr)
# must compute
# TODO precompute as much as possible
reps = self.bdd_reps[node_no_bft, :].reshape(-1, 1)
try:
ber_args = np.hstack(
# this is p1
(
(self.bdd_param[plate_idx][:, node_no_bft] * beta[:, i_high, 1] / beta[:, i_curr, 1]).reshape(
-1, 1
),
# stacked with reps
reps,
)
)
except Exception as e:
# print len(h_nodes)
# print self.bdd_param[0].shape
# print plate_idx
# print node_no_bft
# print i_high
# print beta[:,i_curr,1]
logd(e)
samples = np.apply_along_axis(lambda x: np.random.multinomial(x[1], [x[0], None]), 1, ber_args)
# TODO figure out how to update self.x with vectorization
# idea: group by i in (i,j,k) and work with the matrices separately
# print self.x
np.apply_along_axis(
self.update_x_high_count, 1, np.hstack((pb_plate.plate[:, node_no_bft], samples[:, [0]])).astype(int)
)
np.apply_along_axis(
self.update_x_low_count, 1, np.hstack((pb_plate.plate[:, node_no_bft], samples[:, [1]])).astype(int)
)
start_upd_reps = time.clock()
try:
node_no_high = h_nodes.index(h_high)
self.bdd_reps[node_no_high, :] += samples[:, [0]].reshape(-1)
except:
pass # leaf node
try:
node_no_low = h_nodes.index(h_low)
self.bdd_reps[node_no_low, :] += samples[:, [1]].reshape(-1)
except:
pass # leaf node
end_upd_reps = time.clock()
def PrintProcessSaleCard(retCardInfo):
if retCardInfo == None:
logd("PrintProcessCardResult. No card info")
return
log("%s:" %(retCardInfo.get('Type', "Unknown card type")))
printResFormat(" ExpDate", retCardInfo.get('ExpDate', None), False)
printResFormat(" Name", retCardInfo.get('Name', None), True)
printResFormat(" LastName", retCardInfo.get('LastName', None), False)
printResFormat(" Info", retCardInfo.get('Info', None), False)
printResFormat(" Card num", retCardInfo.get('UID', None), False)
printResFormat(" Type", retCardInfo.get('Type', None), False)
printResFormat(" State", retCardInfo.get('State', None), False)
printResFormat(" Time", retCardInfo.get('EnterTime', None), True)
printResFormat(" Price(UAH)", retCardInfo.get('Price', None), True)
def parse(self):
"""
Main function. Called by __init__.
:rtype: None
"""
try:
self.parse_probs()
#n_plate_file = os.path.join(self.dir_file, 'n_plates')
#with open(n_plate_file, 'r') as fin:
# n_plates = int(fin.read().strip())
self.plates = [PBPlate(*self.parse_plate(f_name))
for f_name in os.listdir(self.dir_file) if f_name.endswith('.plate')]
except IOError as e:
logd('Parsing failed!')
logging.exception('Parsing failed: '+str(e))
def main():
N = 5000
c1 = bernoulli.rvs(p1, size=N)
c2 = bernoulli.rvs(p2, size=N)
c3 = bernoulli.rvs(p3, size=N)
res = (~c1 & c2) | (c1 & c3)
T = np.sum(res)
F = np.size(res) - np.sum(res)
logd("{}/{} \t {}".format(T, N, T / float(N)))
df = pd.DataFrame(np.c_[c1, res, range(N)])
g = df.groupby([0, 1])[2].apply(lambda x: len(x.unique()))
with open("coins2_artificial.pb", "w") as fout:
fout.write(
"""
% aprob debug flags
%:- set_value(dbg_read,2).
%:- set_value(dbg_query,2).
%:- set_value(dbg_write,2).
"""
)
for i, j in reversed(zip(g, g.keys())):
j_str = [2 if el == 1 else 1 for el in j]
fout.write("observe({}, {}, {}).\n".format(j_str[0], j_str[1], i))
fout.write(
"""
pb_dirichlet(1.0, toss, 2, 3).
generate(1, Val) :-
toss(1, 1),
toss(Val, 2).
generate(2, Val) :-
toss(2, 1),
toss(Val, 3).
pb_plate(
[observe(Val1, Val2, Count)],
Count,
[generate(Val1, Val2)]).
"""
)
def sample_bdd_plate2(self, pb_plate, beta, plate_idx):
bdd = pb_plate.bdd
reps = pb_plate.reps
NNodes = bdd.shape[0]
NParamSets = reps.shape[0]
self.bdd_reps = np.zeros((NNodes, NParamSets))
self.bdd_reps[-1, :] = reps
for i in range(bdd.shape[0] - 1, -1, -1):
bdd_node = bdd[i]
if bdd_node[0] == -1:
continue
# must compute
# TODO precompute as much as possible
reps = self.bdd_reps[i, :].reshape(-1, 1)
try:
ber_args = np.hstack(
# this is p1
(
(self.bdd_param[plate_idx][:, bdd_node[0]] * beta[:, bdd_node[1], 1] / beta[:, i, 1]).reshape(
-1, 1
),
# stacked with reps
reps,
)
)
except Exception as e:
logd(e)
logd(ber_args)
samples = np.apply_along_axis(lambda x: np.random.multinomial(x[1], [x[0], None]), 1, ber_args)
# TODO figure out how to update self.x with vectorization
# idea: group by i in (i,j,k) and work with the matrices separately
# print self.x
np.apply_along_axis(
self.update_x_high_count, 1, np.hstack((pb_plate.plate[:, bdd_node[0]], samples[:, [0]])).astype(int)
)
np.apply_along_axis(
self.update_x_low_count, 1, np.hstack((pb_plate.plate[:, bdd_node[0]], samples[:, [1]])).astype(int)
)
start_upd_reps = time.clock()
self.bdd_reps[bdd_node[1], :] += samples[:, [0]].reshape(-1)
self.bdd_reps[bdd_node[2], :] += samples[:, [1]].reshape(-1)
end_upd_reps = time.clock()
def test_coin_categorical(self):
logd('### COIN CATEGORICAL ###')
cmd_str = 'peircebayes ./coins/coin_categorical.pb -d'
PBCoreTests.call_pb(cmd_str)
thetash = np.load('/tmp/peircebayes/avg_samples.npz')
thetash = thetash['arr_0']
thetash = thetash[:, 1]
thetas = [0.5]
logd('Param should be (0.5, 0.5).')
logd('Learned params:\n{}'.format(thetash))
norm2 = np.linalg.norm(thetas-thetash)
logd('Norm2:\n{}'.format(norm2))
self.assertTrue(norm2<0.001)
def PrintProcessEmployeeCard(retCardInfo):
if retCardInfo == None:
logd("PrintProcessCardResult. No card info")
return
log("%s. %s:" %(retCardInfo['EnterExit'], retCardInfo.get('Type', None)))
printResFormat(" ExpDate", retCardInfo.get('ExpDate', None), False)
printResFormat(" Name", retCardInfo.get('Name', None), True)
printResFormat(" LastName", retCardInfo.get('LastName', None), True)
printResFormat(" Info", retCardInfo.get('Info', None), True)
printResFormat(" Card num", retCardInfo.get('UID', None), True)
printResFormat(" Type", retCardInfo.get('Type', None), False)
printResFormat(" State", retCardInfo.get('State', None), False)
printResFormat(" EnterTime", retCardInfo.get('EnterTime', None), True)
printResFormat(" ExitTime", retCardInfo.get('ExitTime', None), True)
secCount = retCardInfo.get('DiffTime', None)
if secCount:
m, s = divmod(secCount, 60)
h, m = divmod(m, 60)
strTimeDelta = "%d:%02d" % (h, m)
printResFormat(" DiffTime(H:MM)", strTimeDelta, True)
def test_lda_cgs(self):
logd('### LDA CGS (depends on LDA2) ###')
cmd_str = ("peircebayes ./lda/lda_example2.pb"+
" -n 150 -b 100 -d -t -a cgs")
PBCoreTests.call_pb(cmd_str)
PBCoreTests.call_pb('cp /tmp/peircebayes/lls.npz lda/lda_amcmc_lls.npz')
with open('lda/lda2_lls.npz', 'r') as f:
lda2_lls = np.load(f)['lls']
with open('lda/lda_amcmc_lls.npz', 'r') as f:
lda_amcmc_lls = np.load(f)['lls']
x = np.arange(lda2_lls.shape[0])
logd('lda2')
logd(lda2_lls)
logd('lda cgs')
logd(lda_amcmc_lls)
plt.figure()
plt.plot(x, lda2_lls, linestyle='-.', color='b', label='PB_ugs')
plt.plot(x, lda_amcmc_lls, linestyle='-', color='r', label='PB_cgs')
plt.legend(loc='lower right')
plt.savefig('lda/lls_ugs_cgs.pdf', format='pdf')
def sample_bdd_plates(self):
"""
Sample x method. Calls :meth:`backward_plates` and for each plate :func:`prob_inference_dev.sample_bdd_plate_cy`. Updates x.
:rtype: None
"""
self.reset_x()
# mem1 = psutil.virtual_memory().percent
# logd('before beta: {}'.format(mem1))
start = time.clock()
betas = self.backward_plates()
end = time.clock()
logd("beta time: {} s".format(end - start))
# mem2 = psutil.virtual_memory().percent
# logd('after beta: {}'.format(mem2))
# start = time.clock()
for (plate_idx, (pb_plate, beta)) in enumerate(zip(self.pb_model.plates, betas)):
self.sample_bdd_plate3(pb_plate, beta, plate_idx)
def total_ll(self):
logbetas = [np.log(beta[:,-1,1]) if hasattr(beta, 'shape') else np.array(beta)
for beta in self.betas]
self.total_ll = 0
duds = 0
for pb_plate in self.pb_model.plates:
if pb_plate.bdd is None:
duds += 1
self.total_ll += np.log(10**-6)
self.total_ll += np.sum(logbetas)
logd('total_ll')
logd(self.total_ll)
logd('duds')
logd(duds)
return self.total_ll
def test_coin_beta_args(self):
logd('### COIN BETA ARGS ###')
cmd_str = 'peircebayes ./coins/coin_beta_args.pb -d'
PBCoreTests.call_pb(cmd_str)
thetash = np.load('/tmp/peircebayes/avg_samples.npz')
thetash = thetash['arr_0']
thetash = thetash[:, 1]
logd('Param should be Beta(15, 85) distributed.')
logd('Learned params:\n{}'.format(thetash))
def sample(self, n, burn_in, lag, track_ll, metric, in_file, algo, dbg_percent=5.0):
dbg_iter = np.round((n+burn_in)*dbg_percent/100)
metrics = {
'joint' : self.joint_collapsed,
'lda' : self.likelihood_lda,
'total' : self.total_ll,
'joint_un' : self.joint_uncollapsed
}
self.metric = metrics[metric]
algos = {
'ugs' : lambda x:
(lambda: None, x.ugs_iteration, x.ugs_samples, x.ugs_lls),
'amcmc' : lambda x:
(lambda: None, x.amcmc_iteration, x.ugs_samples, x.ugs_lls),
'cgs' : lambda x:
(x.cgs_init, x.cgs_iteration, x.cgs_samples, x.ugs_lls),
'predict' : lambda x:
(lambda : None, x.predict_iteration, x.predict_samples, None)
}
start_sampling = time.time()
self.generic_sampler(n, burn_in, lag, track_ll, in_file, dbg_iter,
*algos[algo](self))
end_sampling = time.time()
logd('Sampling time: {} s '.format(end_sampling-start_sampling))
def test_rim2(self):
logd('### RIM2 ###')
cmd_str = 'peircebayes ./rim/rim_example2.pb -d'
PBCoreTests.call_pb(cmd_str)
from rim.sample_mallows import p2,p3,p4
p = [p2,p3,p4]
thetas = np.load('/tmp/peircebayes/avg_samples.npz')
ph2 = thetas['arr_0'][-1]
ph3 = thetas['arr_1'][-1]
ph4 = thetas['arr_2'][-1]
ph = [ph2, ph3, ph4]
logd('Original params:\n{}'.format('\n'.join([str(np.array(pi))
for pi in p])))
logd('Learned params:\n{}'.format('\n'.join([str(phi)
for phi in ph])))
norm2 = np.average([ np.linalg.norm(pi-pih) for pi,pih in zip(p,ph) ])
logd('Norm2:\n{}'.format(norm2))
self.assertTrue(norm2<0.1)
def test_coins2_cgs(self):
logd('### COINS2 CGS ###')
cmd_str = 'peircebayes ./coins/coins2_example.pb -d -a cgs'
PBCoreTests.call_pb(cmd_str)
with open('/tmp/peircebayes/aprob/plates/out1.plate', 'r') as fin:
out1 = fin.read()
with open('coins/test2.plate', 'r') as fin:
test_out = fin.read()
self.assertEqual(out1, test_out)
# ground truth
from coins.coins2_gen import p1,p2,p3
thetas = np.array([p1, p2, p3])
# learned params
thetash = np.load('/tmp/peircebayes/avg_samples.npz')
thetash = thetash['arr_0']
thetash = thetash[:, 1]
logd('Original params:\n{}'.format(thetas))
logd('Learned params:\n{}'.format(thetash))
norm2 = np.linalg.norm(thetas-thetash)
logd('Norm2:\n{}'.format(norm2))
self.assertTrue(norm2<0.1)
def test_lda1(self):
logd('### LDA1 ###')
cmd_str = 'peircebayes ./lda/lda_example.pb -d'
PBCoreTests.call_pb(cmd_str)
with open('/tmp/peircebayes/aprob/plates/out1.plate', 'r') as fin:
out1 = fin.read()
with open('lda/lda_example_test.plate', 'r') as fin:
test_out = fin.read()
self.assertEqual(out1, test_out)
thetash = np.load('/tmp/peircebayes/avg_samples.npz')
muh = thetash['arr_0']
phih = thetash['arr_1']
logd('Learned mu:\n{}'.format(muh))
logd('Learned phi:\n{}'.format(phih))
def test_lda2(self):
logd('### LDA2 ###')
cmd_str = ('peircebayes ./lda/lda_example2.pb'+
' -n 150 -b 100 -d -t')
PBCoreTests.call_pb(cmd_str)
# original params
# phi is given as the horizontal and vertical topics
# on 5X5 images
# word vocabulary
W = 25
# image size
L = int(np.sqrt(W))
# topics
T = 2*L
phi = [np.zeros((L, L)) for i in range(T)]
line = 0
for phi_t in phi:
if line >= L:
trueLine = int(line - L)
phi_t[:,trueLine] = 1./L*np.ones(L)
else:
phi_t[line] = 1./L*np.ones(L)
line += 1
# plot original params
PBOptTests.plot_topics(T, phi, 'lda2_phi')
phi_flat = np.array(phi).reshape((T,W))
thetash = np.load('/tmp/peircebayes/avg_samples.npz')
muh = thetash['arr_0']
phih_flat = thetash['arr_1']
logd('Learned mu shape:\n{}'.format(muh.shape))
logd('Learned phi shape:\n{}'.format(phih_flat.shape))
phih = [phi_t.reshape(L,L) for phi_t in list(phih_flat)]
# plot learned params
PBOptTests.plot_topics(T, phih, 'lda2_phih')
norm2 = np.average(np.apply_along_axis(np.linalg.norm, 1,
phi_flat-phih_flat))
logd('Average Norm2 on phi:\n{}'.format(norm2))
self.assertTrue(norm2 < 1)
PBCoreTests.call_pb('cp /tmp/peircebayes/lls.npz lda/lda2_lls.npz')
def generic_sampler(self, n, burn_in, lag, track_ll, in_file, dbg_iter,
init_f, iteration_f, post_process_samples_f, post_process_lls_f):
'''
samples = list
track_ll = boolean
Signature for functions:
- init_f() -> None
- iteration_f(track_ll) -> sample, ll
- post_process_samples_f(samples) -> None
- post_process_lls_f(samples) -> None
'''
max_iter = n+burn_in
samples = []
if track_ll:
lls = []
init_f()
logd('Done init!')
for i in range(max_iter):
start_it = time.time()
sample, ll = iteration_f(track_ll)
end_it = time.time()
#logd('time: {}'.format(end_it-start_it))
#logd(np.sum(self.x[0], axis=1))
#logd(np.sum(self.x[1], axis=0))
#logd(np.sum(self.x[1]))
#logd(ll)
if np.isclose(i%dbg_iter, 0):
logd('iteration {} time: {} s'.format(i, end_it-start_it))
#logd('nbdds')
#logd(len(self.pb_model.plates))
#logd(ll)
if i>=burn_in and (i-burn_in)%lag==0:
samples.append(sample)
if track_ll:
lls.append(ll)
logd('Number of samples: {}'.format(len(samples)))
post_process_samples_f(samples)
if track_ll and post_process_lls_f:
post_process_lls_f(lls, in_file)
def compile_k_parallel(pb_model, debug, tmp_dir):
"""
Adds a bdd attribute of type :class:`BDD` to each plate in a :class:`formula_gen.PBModel`. If f_str is empty, the BDD will be None.
:param pb_model:
:type pb_model: :class:`formula_gen.PBModel`
:rtype: None
"""
logd('parallel_kc')
start = time.time()
mgr = pycudd.DdManager()
mgr.SetDefault()
mgrPickable = PickalableSwigPyObject(mgr)
n_cores = multiprocessing.cpu_count()
batch_size = int(float(len(pb_model.plates))/float(n_cores))
l = [copy.copy(pb_plate) for pb_plate in pb_model.plates]
bdds = Parallel(n_jobs=n_cores, batch_size=batch_size)(delayed(create_bdd)(mgrPickable, pb_plate, debug, tmp_dir) for pb_plate in l)
logd('parallel_kc0 took: {}'.format(time.time()-start))
for bdd,pb_plate in zip(bdds, pb_model.plates):
pb_plate.bdd = bdd
logd('parallel_kc took: {}'.format(time.time()-start))
def ProcessCoworkerCard(newCard):
"""
newCard { 'uid' : 'time' } - time when card was detected
ListofCheckin [ newCard, newCard, ...., newCard ]
"""
global ListofCheckin
logd("ProcessCoworkerCard. Enter: %s" %(newCard.keys()[0]))
uid = newCard.keys()[0]
curCard = database.GetCard(uid)
retCardInfo = {}
retCardInfo['UID'] = curCard['UID']
retCardInfo['Type'] = curCard['Type']
retCardInfo['Name'] = curCard['Name']
retCardInfo['LastName'] = curCard['LastName']
retCardInfo['State'] = curCard['State']
retCardInfo['ExpDate'] = curCard['ExpDate']
retCardInfo['Info'] = curCard['Info']
retCardInfo['EnterTime'] = None
retCardInfo['ExitTime'] = None
retCardInfo['DiffTime'] = None
retCardInfo['Price'] = None
retCardInfo['EnterExit'] = "Unknown"
if uid in ListofCheckin:
retCardInfo['EnterExit'] = "Exit"
enterTime = ListofCheckin[uid]
exitTime = newCard[uid]
delta = int((exitTime - enterTime).total_seconds()+0.5)*MultiplyTimeForDebug
retCardInfo['EnterTime'] = enterTime.strftime(timeFormat)
retCardInfo['ExitTime'] = exitTime.strftime(timeFormat)
retCardInfo['DiffTime'] = delta
logd("Card %s:" % (uid))
logd(" Enter time: %s" % (retCardInfo['EnterTime']))
logd(" Exit time: %s" % (retCardInfo['ExitTime']))
logd(" Diff time: %s sec" %(retCardInfo['DiffTime']))
del ListofCheckin[uid]
else:
retCardInfo['EnterExit'] = "Enter"
ListofCheckin.update(newCard)
retCardInfo['EnterTime'] = ListofCheckin[uid].strftime(timeFormat)
logd("New card: %s" % (uid))
logd("Enter time: %s" % (retCardInfo['EnterTime']))
logd("")
return retCardInfo
def gibbs_sampler_plates(self, n, burn_in, lag, track_ll, in_file, metric, algo):
"""
Gibbs sampling. Updates x and theta, creates a theta_avg attribute and optionally saves metric to an output file.
:param n:
:type n: see '-n' in :ref:`cl`
:param burn_in:
:type burn_in: see '-b' in :ref:`cl`
:param lag:
:type lag: see '-l' in :ref:`cl`
:param track_ll:
:type track_ll: see '-t' in :ref:`cl`
:param in_file:
:type in_file: see 'in_file' in :ref:`cl`
:param metric:
:type metric: see '-m' in :ref:`cl`
:param algo:
:type algo: see '-a' in :ref:`cl`
:rtype: None
"""
start_gibbs = time.time()
metrics = {"joint": lambda x: x.joint_collapsed(), "lda_ll": lambda x: x.likelihood_lda()}
metric_f = metrics[metric]
# ngc = gc.collect()
# logd('Unreachable objects:{}'.format(ngc))
n = n + burn_in # n is total number of iterations
thetas = []
if track_ll:
lls = []
self.first_sample = None
# main loop
for i in range(n):
logd("Sampling iteration: {}".format(i))
if algo == "ungibbs":
start = time.clock()
# SAMPLE THETA
# logd('SAMPLE THETA')
self.sample_theta()
if not self.first_sample:
self.first_sample = self.theta
end = time.clock()
logd("sample theta time: {} s".format(end - start))
# logd('theta')
# logd(self.theta)
thetas.append(self.theta)
# SAMPLE X
# logd('SAMPLE X')
start = time.clock()
self.sample_bdd_plates()
end = time.clock()
logd("sample x time: {} s".format(end - start))
elif algo == "amcmc":
start = time.clock()
# SAMPLE THETA
# logd('SAMPLE THETA')
self.set_theta_mean()
if not self.first_sample:
self.first_sample = self.theta
end = time.clock()
logd("sample theta time: {} s".format(end - start))
# logd('theta')
# logd(self.theta)
thetas.append(self.theta)
# SAMPLE X
# logd('SAMPLE X')
start = time.clock()
self.sample_bdd_plates()
end = time.clock()
logd("sample x time: {} s".format(end - start))
if track_ll:
lls.append(metric_f(self))
# logd('x')
# logd(self.x)
# self.plot_phi(theta)
end_gibbs = time.time()
logd("Sampling time: {} s ".format(end_gibbs - start_gibbs))
# POST PROCESS
if burn_in is not None and int(burn_in) >= 0:
thetas = thetas[burn_in:]
if lag is not None and int(lag) > 0:
thetas = [theta for i, theta in enumerate(thetas) if i % lag == 0]
logd("No. of samples: {}".format(len(thetas)))
# avg_theta
sum_thetas = [np.zeros(theta_d.shape) for theta_d in self.theta]
for theta_it in thetas:
for i, theta_d in enumerate(theta_it):
sum_thetas[i] += theta_d
self.theta_avg = [theta_d / float(len(thetas)) for theta_d in sum_thetas]
logd("theta avg")
for theta_it in self.theta_avg:
logd(theta_it)
# ll tracking post-process
if track_ll:
self.plot_ll(lls, burn_in, in_file)
np.savez("/tmp/peircebayes/lls", **{"lls": np.array(lls)})
def parse_plate(self, f_name):
"""
Parses a plate file. Returns a tuple of plate, reps, kid, cat_list, f_str to create a :class:`PBPlate` object. Called by :func:`parse`.
:param i: index of the plate
:type i: int
:rtype: tuple
"""
#plate_file = os.path.join(self.dir_file,'out{}.plate'.format(ii))
plate_file = os.path.join(self.dir_file, f_name)
logd(plate_file)
with open(plate_file, 'r') as fin:
plate = []
kid = []
reps = []
cat_list = []
for line in fin:
line_split = line.split(';')
line_sols, line_reps = line_split[:-1], line_split[-1]
reps.append(int(line_reps))
cat_d = {}
plate_row = set()
for line_idxs in line_sols:
for idxs in line_idxs.split('.'):
i, j, k = [int(idx) for idx in idxs.split(',')]
if cat_d.has_key((i,j)):
if k not in cat_d[(i,j)]:
cat_d[(i,j)].append(k)
else:
cat_d[(i,j)] = [k]
plate_row.add((i,j,k))
# sort plate_row
plate_row = sorted(list(plate_row),
key = lambda x: (x[0], x[1], x[2]))
# sort cat_d and remove last choices from plate_row
# if necessary (>1 choice)
# however, keep a copy of plate_row to generate
# the formula from
plate_row_f = copy.copy(plate_row)
for (i,j) in cat_d:
cat_d[(i,j)] = sorted(cat_d[(i,j)])
if len(cat_d[(i,j)])>1:
last_k = cat_d[(i,j)][-1]
plate_row.remove((i, j, last_k))
plate.append(plate_row)
cat_list.append(cat_d)
kid_row = []
for i,j,k in plate_row:
kid_row.append(cat_d[(i,j)].index(k))
kid.append(kid_row)
# convert to numpy
plate = np.array(plate)
reps = np.array(reps)
kid = np.array(kid)
#logd('=== i: {} ==='.format(ii))
f_str = self.bool_gen(line_sols, plate_row_f, cat_d)
#logd(plate.shape)
#logd(reps.shape)
#logd(kid.shape)
#logd(len(cat_list))
#logd(plate)
#logd(reps)
#logd(kid)
#logd(cat_list)
#logd('f_str')
#logd(f_str)
return plate, reps, kid, cat_list, f_str
def peircebayes():
"""
Main function. Parses args, then runs the peircebayes pipeline:
1. :func:`logical_inference`
2. :func:`parse_and_compile`
3. :func:`probabilistic_inference`
:rtype: None
"""
cl_args = cl_parse().parse_args()
args = json.load(cl_args.in_file) if cl_args.config else cl_args
if not args.in_file and not args.learning:
print("You need to specify an input file for abduction!")
return
tmp_dir = args.output
if not args.learning:
rewrite_dir(tmp_dir)
if args.debug:
logging.basicConfig(filename=os.path.join(tmp_dir, "pb.log"), level=logging.DEBUG)
np.random.seed(args.seed)
infer_options = {
"n": args.n,
"burn_in": args.burn,
"lag": args.lag,
"track_ll": args.track,
"metric": args.metric,
"in_file": args.in_file,
"algo": args.algo,
}
if not args.learning:
if args.wrapper == "extension":
ext = os.path.splitext(args.in_file)[1][1:]
else:
ext = args.wrapper
ext_d = {
"abduction": lambda x: l_abduction(*x),
"pb": lambda x: l_abduction(*x),
"pl": lambda x: l_abduction(*x),
"py": lambda x: l_py(*x),
"asp": lambda x: l_asp(*x),
}
if ext in ext_d:
logical_inference = ext_d[ext]
else:
print("Wrong file extension!" + "See help (-h) on the -w option for supported extensions.")
return
logd("### Started Logical Inference ###")
t_li_start = time.time()
logical_inference((args.in_file, tmp_dir, args.output))
t_li_end = time.time()
logd("Finished Logical Inference in:\n{} seconds".format(t_li_end - t_li_start))
if not args.abduction:
logd("### Started Knowledge Compilation ###")
t_kc_start = time.time()
model = parse_and_compile(args.debug, tmp_dir, args.pkc)
t_kc_end = time.time()
logd("Finished Knowledge Compilation in:\n{} seconds".format(t_kc_end - t_kc_start))
logd("### Started Probabilistic Inference ###")
t_pi_start = time.time()
probabilistic_inference(model, infer_options, tmp_dir)
t_pi_end = time.time()
logd("Finished Probabilistic Inference in:\n{} seconds".format(t_pi_end - t_pi_start))
def ProcessSaleCard(newCard):
"""
newCard { 'uid' : 'time' } - time when card was detected
ListofCheckin [ newCard, newCard, ...., newCard ]
"""
logd("ProcessSaleCard. Time: %s" %(newCard.values()[0]))
uid = newCard.keys()[0]
time = newCard.values()[0]
curCard = database.GetCard(uid)
retCardInfo = {}
retCardInfo['UID'] = curCard['UID']
retCardInfo['Type'] = curCard['Type']
retCardInfo['Name'] = curCard['Name']
retCardInfo['LastName'] = curCard['LastName']
retCardInfo['State'] = curCard['State']
retCardInfo['ExpDate'] = curCard['ExpDate']
retCardInfo['Info'] = curCard['Info']
retCardInfo['EnterTime'] = None
retCardInfo['ExitTime'] = None
retCardInfo['DiffTime'] = None
retCardInfo['Price'] = getSalePrice(retCardInfo['Name'])
retCardInfo['EnterExit'] = None
retCardInfo['EnterTime'] = time.strftime(timeFormat)
retCardInfo['ExitTime'] = time.strftime(timeFormat)
retCardInfo['DiffTime'] = 0
logd("Card %s:" % (uid))
logd(" Enter time: %s" % (retCardInfo['EnterTime']))
logd(" Exit time: %s" % (retCardInfo['ExitTime']))
logd(" Diff time: %s sec" %(retCardInfo['DiffTime']))
logd("")
return retCardInfo
def printCardInfo(MyCard):
if MyCard == None:
return
logd("UID : %s" % (MyCard['UID']))
logd("Type : %s" % (MyCard['Type']))
logd("Name : %s" % (MyCard['Name']))
logd("LastName : %s" % (MyCard['LastName']))
logd("State : %s" % (MyCard['State']))
logd("ExpDate : %s" % (MyCard['ExpDate']))
logd("Info : %s" % (MyCard['Info']))
