def __buildMaskPyramid(self, mask, pyramid):
"""Given a mask and a pyramid of masks makes a pyramid, where it uses the or operation for combining flags."""
code = start_cpp() + """
// Make curr all false...
for (int y=0;y<Ncurr[0];y++)
{
for (int x=0;x<Ncurr[1];x++) CURR2(y,x) = 0;
}
// Iterate prev, and update curr...
for (int y=0;y<Nprev[0];y++)
{
for (int x=0;x<Nprev[1];x++)
{
if (PREV2(y,x)!=0)
{
CURR2(y/2,x/2) = 1;
}
}
}
"""
pyramid[0][:, :] = mask
for l in xrange(1, len(pyramid)):
prev = pyramid[l - 1]
curr = pyramid[l]
weave.inline(code, ['prev', 'curr'])
def test_size_gc(self):
code = start_cpp(linked_list_gc_code) + """
int errors = 0;
ListRef<> wibble;
if (wibble.Size()!=0) errors += 1;
ItemRef<> * it = wibble.Append();
if (wibble.Size()!=1) errors += 1;
if (wibble.RefTotal()!=0) errors += 1;
it->IncRef();
it->IncRef();
if (it->RefCount()!=2) errors += 1;
if (wibble.RefTotal()!=2) errors += 1;
it->DecRef();
it->DecRef();
if (wibble.RefTotal()!=0) errors += 1;
if (wibble.Size()!=0) errors += 1;
return_val = errors;
"""
errors = weave.inline(code, support_code=linked_list_gc_code)
self.assertEqual(errors,0)
def testCodeC(self, name, exemplar_list):
ret = start_cpp() + """
bool %(name)s(PyObject * data, void * test, size_t test_length, int exemplar)
{
int * feature = (int*)test;
float * plane_axis = (float*)test + %(dims)i;
float offset = *((float*)test + %(dims)i*2);
%(channelType)s channel = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
float value = 0.0;
for (int i=0;i<%(dims)i;i++)
{
float v = (float)%(channelName)s_get(channel, exemplar, feature[i]);
value += v*plane_axis[i];
}
return (value-offset)>=0.0;
}
""" % {
'name': name,
'channel': self.channel,
'channelName': exemplar_list[self.channel]['name'],
'channelType': exemplar_list[self.channel]['itype'],
'dims': self.dims
}
return ret
def test_loop(self):
extra = """
struct Number
{
int num;
};
"""
code = start_cpp(linked_list_code+extra) + """
int errors = 0;
List<Number> wibble;
for (int i=0;i<10;i++)
{
Item<Number> * it = wibble.Append();
it->num = i;
}
if (wibble.Size()!=10) errors += 1;
int i = 0;
for (Item<Number> * targ = wibble.First(); targ->Valid(); targ = targ->Next())
{
if (i!=targ->num) errors += 1;
i += 1;
}
return_val = errors;
"""
errors = weave.inline(code, support_code=linked_list_code+extra)
self.assertEqual(errors,0)
def test_loop(self):
extra = """
struct Number
{
int num;
};
"""
code = start_cpp(linked_list_code + extra) + """
int errors = 0;
List<Number> wibble;
for (int i=0;i<10;i++)
{
Item<Number> * it = wibble.Append();
it->num = i;
}
if (wibble.Size()!=10) errors += 1;
int i = 0;
for (Item<Number> * targ = wibble.First(); targ->Valid(); targ = targ->Next())
{
if (i!=targ->num) errors += 1;
i += 1;
}
return_val = errors;
"""
errors = weave.inline(code, support_code=linked_list_code + extra)
self.assertEqual(errors, 0)
def __buildMaskPyramid(self, mask, pyramid):
"""Given a mask and a pyramid of masks makes a pyramid, where it uses the or operation for combining flags."""
code = start_cpp() + """
// Make curr all false...
for (int y=0;y<Ncurr[0];y++)
{
for (int x=0;x<Ncurr[1];x++) CURR2(y,x) = 0;
}
// Iterate prev, and update curr...
for (int y=0;y<Nprev[0];y++)
{
for (int x=0;x<Nprev[1];x++)
{
if (PREV2(y,x)!=0)
{
CURR2(y/2,x/2) = 1;
}
}
}
"""
pyramid[0][:,:] = mask
for l in xrange(1,len(pyramid)):
prev = pyramid[l-1]
curr = pyramid[l]
weave.inline(code, ['prev','curr'])
def test_size_gc(self):
code = start_cpp(linked_list_gc_code) + """
int errors = 0;
ListRef<> wibble;
if (wibble.Size()!=0) errors += 1;
ItemRef<> * it = wibble.Append();
if (wibble.Size()!=1) errors += 1;
if (wibble.RefTotal()!=0) errors += 1;
it->IncRef();
it->IncRef();
if (it->RefCount()!=2) errors += 1;
if (wibble.RefTotal()!=2) errors += 1;
it->DecRef();
it->DecRef();
if (wibble.RefTotal()!=0) errors += 1;
if (wibble.Size()!=0) errors += 1;
return_val = errors;
"""
errors = weave.inline(code, support_code=linked_list_gc_code)
self.assertEqual(errors, 0)
def global_colour_histogram(image, mask, size=256):
"""Given an image ([0:height, 0:width, 0:3 - r,g,b] -> uint8) and a mask ([0:height, 0:width] -> bool.) this returns a histogram of how many in each colour bin belongs to the foreground and background. Return is [0:size - r, 0:size - g, 0:size - b, 0:2 - bg,fg] -> float32. For the default size of 256 this is 128 meg, so be nice! Note that if pixels don't land exactly into a bin they will be linearly interpolated, hence the use of float."""
ret = numpy.zeros((size, size, size, 2), dtype=numpy.float32)
code = start_cpp() + """
float mult_r = (size-1) / 255.0;
float mult_g = (size-1) / 255.0;
float mult_b = (size-1) / 255.0;
int y, x;
for (y=0; y<Nimage[0]; y++)
{
for (x=0; x<Nimage[1]; x++)
{
float r = mult_r * IMAGE3(y, x, 0);
float g = mult_g * IMAGE3(y, x, 1);
float b = mult_b * IMAGE3(y, x, 2);
int lr = int(floor(r));
int lg = int(floor(g));
int lb = int(floor(b));
float tr = r - lr;
float tg = g - lg;
float tb = b - lb;
int hr = int(ceil(r));
int hg = int(ceil(g));
int hb = int(ceil(b));
if (hr>=size) hr = size-1;
if (hg>=size) hg = size-1;
if (hb>=size) hb = size-1;
char ci = (MASK2(y, x)!=0) ? 1 : 0;
RET4(lr, lg, lb, ci) += (1.0-tr) * (1.0-tg) * (1.0-tb);
RET4(hr, lg, lb, ci) += tr * (1.0-tg) * (1.0-tb);
RET4(lr, hg, lb, ci) += (1.0-tr) * tg * (1.0-tb);
RET4(lr, lg, hb, ci) += (1.0-tr) * (1.0-tg) * tb;
RET4(lr, hg, hb, ci) += (1.0-tr) * tg * tb;
RET4(hr, lg, hb, ci) += tr * (1.0-tg) * tb;
RET4(hr, hg, lb, ci) += tr * tg * (1.0-tb);
RET4(hr, hg, hb, ci) += tr * tg * tb;
}
}
"""
weave.inline(code, ['ret', 'image', 'mask'])
return ret
def testCodeC(self, name, exemplar_list):
ret = start_cpp() + """
bool %(name)s(PyObject * data, void * test, size_t test_length, int exemplar)
{
int feature = *(int*)test;
float offset = *((float*)test + 1);
%(channelType)s channel = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
float value = (float)%(channelName)s_get(channel, exemplar, feature);
return (value-offset)>=0.0;
}
"""%{'name':name, 'channel':self.channel, 'channelName':exemplar_list[self.channel]['name'], 'channelType':exemplar_list[self.channel]['itype']}
return ret
def addTrain(self, goal, gen, es, index=slice(None), weights=None, code=None):
"""This allows you to update the nodes with more data, as though it was used for trainning. The actual tests are not affected, only the statistics at each node - part of incrimental learning. You can optionally proivde code generated by the addTrainC method to give it go faster stripes."""
if isinstance(index, slice):
index = numpy.arange(*index.indices(es.exemplars()))
if code != None:
init = (
start_cpp(code)
+ """
if (dummy==0) // To allow for a dummy run.
{
Exemplar * test_set = (Exemplar*)malloc(sizeof(Exemplar)*Nindex[0]);
for (int i=0; i<Nindex[0]; i++)
{
int ind = index[i];
test_set[i].index = ind;
test_set[i].weight = weights[ind];
test_set[i].next = &test_set[i+1];
}
test_set[Nindex[0]-1].next = 0;
addTrain(self, data, test_set);
free(test_set);
}
"""
)
data = es.tupleInputC()
dummy = 1 if self == None else 0
if weights == None:
weights = numpy.ones(es.exemplars(), dtype=numpy.float32)
return weave.inline(init, ["self", "data", "index", "weights", "dummy"], support_code=code)
else:
# Update this nodes stats...
self.stats = goal.updateStats(self.stats, es, index, weights)
# Check if it has children that need updating...
if self.test != None:
# Need to split the index and send it down the two branches, as needed...
res = gen.do(self.test, es, index)
tIndex = index[res == True]
fIndex = index[res == False]
if tIndex.shape[0] != 0:
self.true.addTrain(goal, gen, es, tIndex, weights)
if fIndex.shape[0] != 0:
self.false.addTrain(goal, gen, es, fIndex, weights)
def test_size(self):
code = start_cpp(linked_list) + """
int errors = 0;
List<> wibble;
if (wibble.Size()!=0) errors += 1;
Item<> * it = wibble.Append();
if (wibble.Size()!=1) errors += 1;
it->Suicide();
if (wibble.Size()!=0) errors += 1;
return_val = errors;
"""
errors = weave.inline(code, support_code=linked_list)
self.assertEqual(errors,0)
def test_size(self):
code = start_cpp(linked_list) + """
int errors = 0;
List<> wibble;
if (wibble.Size()!=0) errors += 1;
Item<> * it = wibble.Append();
if (wibble.Size()!=1) errors += 1;
it->Suicide();
if (wibble.Size()!=0) errors += 1;
return_val = errors;
"""
errors = weave.inline(code, support_code=linked_list)
self.assertEqual(errors, 0)
def evaluate(self, out, gen, es, index=slice(None), code=None):
"""Given a set of exemplars, and possibly an index, this outputs the infered stats entities. Requires the generator so it can apply the tests. The output goes into out, a list indexed by exemplar position. If code is set to a string generated by evaluateC it uses that, for speed."""
if isinstance(index, slice):
index = numpy.arange(*index.indices(es.exemplars()))
if isinstance(code, str) and weave != None:
init = (
start_cpp(code)
+ """
if (Nindex[0]!=0)
{
// Create the Exemplar data structure...
Exemplar * test_set = (Exemplar*)malloc(sizeof(Exemplar)*Nindex[0]);
for (int i=0; i<Nindex[0]; i++)
{
test_set[i].index = index[i];
test_set[i].next = &test_set[i+1];
}
test_set[Nindex[0]-1].next = 0;
// Do the work...
evaluate(self, data, test_set, out);
// Clean up...
free(test_set);
}
"""
)
data = es.tupleInputC()
weave.inline(init, ["self", "data", "index", "out"], support_code=code)
return
if self.test == None:
# At a leaf - store this nodes stats object for the relevent nodes...
for val in index:
out[val] = self.stats
else:
# Need to split the index and send it down the two branches, as needed...
res = gen.do(self.test, es, index)
tIndex = index[res == True]
fIndex = index[res == False]
if tIndex.shape[0] != 0:
self.true.evaluate(out, gen, es, tIndex)
if fIndex.shape[0] != 0:
self.false.evaluate(out, gen, es, fIndex)
def testCodeC(self, name, exemplar_list):
ret = start_cpp() + """
bool %(name)s(PyObject * data, void * test, size_t test_length, int exemplar)
{
int feature = *(int*)test;
float offset = *((float*)test + 1);
%(channelType)s channel = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
float value = (float)%(channelName)s_get(channel, exemplar, feature);
return (value-offset)>=0.0;
}
""" % {
'name': name,
'channel': self.channel,
'channelName': exemplar_list[self.channel]['name'],
'channelType': exemplar_list[self.channel]['itype']
}
return ret
def testCodeC(self, name, exemplar_list):
ret = start_cpp() + """
bool %(name)s(PyObject * data, void * test, size_t test_length, int exemplar)
{
size_t steps = test_length>>2;
int * accept = (int*)test;
%(channelType)s channel = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int value = (int)%(channelName)s_get(channel, exemplar, accept[0]);
for (size_t i=1; i<steps; i++)
{
if (accept[i]==value) return true;
}
return false;
}
"""%{'name':name, 'channel':self.channel, 'channelName':exemplar_list[self.channel]['name'], 'channelType':exemplar_list[self.channel]['itype']}
return ret
def nextFrame(self):
# Increase frame number - need to be 1 based for this...
self.frame += 1
if self.frame>=self.start and self.frame<=self.end:
# Fetch the provided mask...
mask = self.video.fetch(self.channel)
# Load the ground truth file...
fn = os.path.join(self.path, 'groundtruth/gt%06i.png'%self.frame)
gt = cv2array(cv.LoadImage(fn))
gt = gt.astype(numpy.uint8)
if len(gt.shape)==3: gt = gt[:,:,0]
# Loop the pixels and analyse each one, summing into the confusion matrix...
code = start_cpp() + """
for (int y=0; y<Ngt[0]; y++)
{
for (int x=0; x<Ngt[1]; x++)
{
if ((ROI2(y,x)!=0)&&(GT2(y,x)!=170))
{
int t = (GT2(y,x)==255)?1:0;
int g = (MASK2(y,x)!=0)?1:0;
CON2(t,g) += 1;
if (GT2(y,x)==50)
{
SHADOW1(g) += 1;
}
}
}
}
"""
roi = self.roi
con = self.con
shadow = self.shadow
weave.inline(code, ['mask', 'gt', 'roi', 'con', 'shadow'])
return self.frame<=self.end
def nextFrame(self):
# Increase frame number - need to be 1 based for this...
self.frame += 1
if self.frame>=self.start and self.frame<=self.end:
# Fetch the provided mask...
mask = self.video.fetch(self.channel)
# Load the ground truth file...
fn = os.path.join(self.path, 'groundtruth/gt%06i.png'%self.frame)
gt = cv2array(cv.LoadImage(fn))
gt = gt.astype(numpy.uint8)
if len(gt.shape)==3: gt = gt[:,:,0]
# Loop the pixels and analyse each one, summing into the confusion matrix...
code = start_cpp() + """
for (int y=0; y<Ngt[0]; y++)
{
for (int x=0; x<Ngt[1]; x++)
{
if ((ROI2(y,x)!=0)&&(GT2(y,x)!=170))
{
int t = (GT2(y,x)==255)?1:0;
int g = (MASK2(y,x)!=0)?1:0;
CON2(t,g) += 1;
if (GT2(y,x)==50)
{
SHADOW1(g) += 1;
}
}
}
}
"""
roi = self.roi
con = self.con
shadow = self.shadow
weave.inline(code, ['mask', 'gt', 'roi', 'con', 'shadow'])
return self.frame<=self.end
def testCodeC(self, name, exemplar_list):
ret = start_cpp() + """
bool %(name)s(PyObject * data, void * test, size_t test_length, int exemplar)
{
int * feature = (int*)test;
float * plane_axis = (float*)test + %(dims)i;
float offset = *((float*)test + %(dims)i*2);
%(channelType)s channel = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
float value = 0.0;
for (int i=0;i<%(dims)i;i++)
{
float v = (float)%(channelName)s_get(channel, exemplar, feature[i]);
value += v*plane_axis[i];
}
return (value-offset)>=0.0;
}
"""%{'name':name, 'channel':self.channel, 'channelName':exemplar_list[self.channel]['name'], 'channelType':exemplar_list[self.channel]['itype'], 'dims':self.dims}
return ret
def testCodeC(self, name, exemplar_list):
# Add the children...
ret = ''
for i, gen in enumerate(self.gens):
ret += gen.testCodeC(name + '_%i'%i, exemplar_list)
# Put in the final test function...
ret += start_cpp()
ret += 'bool %s(PyObject * data, void * test, size_t test_length, int exemplar)\n'%name
ret += '{\n'
ret += 'void * sub_test = ((char*)test) + 1;\n'
ret += 'size_t sub_test_length = test_length - 1;\n'
ret += 'int which = *(unsigned char*)test;\n'
ret += 'switch(which)\n'
ret += '{\n'
for i in xrange(len(self.gens)):
ret += 'case %i: return %s_%i(data, sub_test, sub_test_length, exemplar);\n'%(i, name, i)
ret += '}\n'
ret += 'return 0;\n' # To stop the compiler issuing a warning.
ret += '}\n'
return ret
def mean(self):
"""Returns an estimate of the mean for each value of the multinomial, as an array, given the evidence provided. (Will itself sum to one - a necesary consequence of being an average of points constrained to the simplex."""
code = start_cpp(smp_code) + """
srand48(time(0));
SMP smp(NflagMat[1],NflagMat[0]);
smp.SetFIA(flagMat);
smp.SetSampleCount(sampleCount);
smp.SetPrior(priorMN, priorConc);
smp.Add(power_array);
smp.Mean(out);
"""
flagMat = self.flagMat
sampleCount = self.sampleCount
priorMN = self.priorMN
priorConc = self.priorConc
power = self.power
out = numpy.empty(flagMat.shape[1] ,dtype=numpy.float32)
weave.inline(code, ['flagMat', 'sampleCount', 'priorMN', 'priorConc', 'power', 'out'], support_code=smp_code)
return out
def testCodeC(self, name, exemplar_list):
# Add the children...
ret = ''
for i, gen in enumerate(self.gens):
ret += gen.testCodeC(name + '_%i' % i, exemplar_list)
# Put in the final test function...
ret += start_cpp()
ret += 'bool %s(PyObject * data, void * test, size_t test_length, int exemplar)\n' % name
ret += '{\n'
ret += 'void * sub_test = ((char*)test) + 1;\n'
ret += 'size_t sub_test_length = test_length - 1;\n'
ret += 'int which = *(unsigned char*)test;\n'
ret += 'switch(which)\n'
ret += '{\n'
for i in xrange(len(self.gens)):
ret += 'case %i: return %s_%i(data, sub_test, sub_test_length, exemplar);\n' % (
i, name, i)
ret += '}\n'
ret += 'return 0;\n' # To stop the compiler issuing a warning.
ret += '}\n'
return ret
def testCodeC(self, name, exemplar_list):
ret = start_cpp() + """
bool %(name)s(PyObject * data, void * test, size_t test_length, int exemplar)
{
size_t steps = test_length>>2;
int * accept = (int*)test;
%(channelType)s channel = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int value = (int)%(channelName)s_get(channel, exemplar, accept[0]);
for (size_t i=1; i<steps; i++)
{
if (accept[i]==value) return true;
}
return false;
}
""" % {
'name': name,
'channel': self.channel,
'channelName': exemplar_list[self.channel]['name'],
'channelType': exemplar_list[self.channel]['itype']
}
return ret
# Copyright (c) 2011, Tom SF Haines
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from utils.start_cpp import start_cpp
# Provides code for estimating the Dirichlet distribution from which a number of multinomial distributions were drawn from, given those multinomials...
dir_est_code = start_cpp() + """
// Defined as a class - you then add each multinomial before requesting a maximum likelihood update of the Dirichlet distribution. It uses Newton-Raphson iterations, and so needs a starting point - you provide a vector to be updated, which can of course save time if it is already close...
class EstimateDir
{
public:
EstimateDir(int vecSize):size(vecSize), samples(0), meanLog(new double[vecSize]), grad(new double[vecSize]), qq(new double[vecSize])
{
for (int i=0;i<vecSize;i++) meanLog[i] = 0.0;
}
~EstimateDir() {delete[] meanLog; delete[] grad; delete[] qq;}
void Add(float * mn)
{
samples += 1;
for (int i=0;i<size;i++)
{
meanLog[i] += (log(mn[i]) - meanLog[i]) / double(samples);
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from scipy import weave
import unittest
from utils.start_cpp import start_cpp
# Defines code for a doubly linked list - simple but works as expected... (Includes its data via templated inheritance - a little strange, but neat and saves on memory thrashing.)
linked_list_code = start_cpp() + """
// Predefinitions...
template <typename ITEM, typename BODY> class Item;
template <typename ITEM, typename BODY> class List;
// Useful default...
struct Empty {};
// Item for the linked list data structure - simply inherits extra data stuff...
template <typename ITEM = Empty, typename BODY = Empty>
class Item : public ITEM
{
public:
Item(List<ITEM,BODY> * head):head(head),next(this),prev(this) {}
def genCodeC(self, name, exemplar_list):
code = ''
states = []
for i, gen in enumerate(self.gens):
c, s = gen[0].genCodeC(name+'_%i'%i, exemplar_list)
code += c
states.append(s)
code += start_cpp() + """
struct State%(name)s
{
void * test;
size_t length;
"""%{'name':name}
for i,s in enumerate(states):
code += ' %s gen_%i;\n'%(s,i)
code += start_cpp() + """
int upto;
int * seq; // Sequence of things to try.
};
void %(name)s_init(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
state.test = 0;
state.length = 0;
state.upto = -1;
state.seq = (int*)malloc(sizeof(int)*%(draws)i);
for (int i=0;i<%(draws)i;i++)
{
float weight = drand48();
"""%{'name':name, 'draws':self.draws, 'count':len(self.gens)}
total = sum(map(lambda g: g[1], self.gens))
ssf = 0.0
for i,gen in enumerate(self.gens):
ssf += gen[1]/total
code += start_cpp() + """
if (weight<%(thres)f) state.seq[i] = %(i)i;
else
"""%{'i':i, 'thres':ssf}
code += start_cpp() + """
state.seq[i] = %(count)i-1;
}
}
bool %(name)s_next(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
while (state.upto<%(draws)i)
{
if (state.upto!=-1)
{
switch (state.seq[state.upto])
{
"""%{'name':name, 'draws':self.draws, 'count':len(self.gens)}
for i in xrange(len(self.gens)):
code += start_cpp() + """
case %(i)i:
if (%(name)s_%(i)i_next(state.gen_%(i)i, data, test_set))
{
state.length = 1 + state.gen_%(i)i.length;
state.test = realloc(state.test, state.length);
((unsigned char*)state.test)[0] = %(i)i;
memcpy((unsigned char*)state.test+1, state.gen_%(i)i.test, state.gen_%(i)i.length);
return true;
}
break;
"""%{'name':name, 'i':i}
code += start_cpp() + """
}
}
state.upto++;
if (state.upto<%(draws)i)
{
switch(state.seq[state.upto])
{
""" %{'draws':self.draws}
for i in xrange(len(self.gens)):
code += start_cpp() + """
case %(i)i:
%(name)s_%(i)i_init(state.gen_%(i)i, data, test_set);
break;
"""%{'name':name, 'i':i}
code += start_cpp() + """
}
}
}
free(state.test);
free(state.seq);
return false;
}
"""
return (code, 'State'+name)
def genCodeC(self, name, exemplar_list):
code = ''
states = []
for i, gen in enumerate(self.gens):
c, s = gen.genCodeC(name+'_%i'%i, exemplar_list)
code += c
states.append(s)
code += start_cpp() + """
struct State%(name)s
{
void * test;
size_t length;
"""%{'name':name}
for i,s in enumerate(states):
code += ' %s gen_%i;\n'%(s,i)
code += start_cpp() + """
int upto;
};
void %(name)s_init(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
state.test = 0;
state.length = 0;
"""%{'name':name}
for i in xrange(len(self.gens)):
code += '%(name)s_%(i)i_init(state.gen_%(i)i, data, test_set);\n'%{'name':name, 'i':i}
code += start_cpp() + """
state.upto = 0;
}
bool %(name)s_next(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
switch (state.upto)
{
"""%{'name':name}
for i in xrange(len(self.gens)):
code += start_cpp() + """
case %(i)i:
if (%(name)s_%(i)i_next(state.gen_%(i)i, data, test_set))
{
state.length = 1 + state.gen_%(i)i.length;
state.test = realloc(state.test, state.length);
((unsigned char*)state.test)[0] = %(i)i;
memcpy((unsigned char*)state.test+1, state.gen_%(i)i.test, state.gen_%(i)i.length);
return true;
}
else state.upto += 1;
"""%{'name':name, 'i':i}
code += start_cpp() + """
}
free(state.test);
return false;
}
"""
return (code, 'State'+name)
def mask_to_sdf(self, mask):
# Initalise with effective infinities...
sdf = numpy.empty(mask.shape, dtype=numpy.float32)
sdf[:, :] = 1e64
# Mark all pixels that are at a transition boundary with the relevant cost - first the diagonals, then the halfs, as half is less than sqrt(2)...
tran_sqrt2 = numpy.zeros(sdf.shape, dtype=numpy.bool)
numpy.logical_or(mask[1:, 1:] != mask[:-1, :-1], tran_sqrt2[:-1, :-1],
tran_sqrt2[:-1, :-1])
numpy.logical_or(mask[1:, :-1] != mask[:-1, 1:], tran_sqrt2[:-1, 1:],
tran_sqrt2[:-1, 1:])
numpy.logical_or(mask[:-1, 1:] != mask[1:, :-1], tran_sqrt2[1:, :-1],
tran_sqrt2[1:, :-1])
numpy.logical_or(mask[:-1, :-1] != mask[1:, 1:], tran_sqrt2[1:, 1:],
tran_sqrt2[1:, 1:])
sdf[tran_sqrt2] = numpy.sqrt(2.0)
tran_half = numpy.zeros(sdf.shape, dtype=numpy.bool)
numpy.logical_or(mask[1:, :] != mask[:-1, :], tran_half[:-1, :],
tran_half[:-1, :])
numpy.logical_or(mask[:-1, :] != mask[1:, :], tran_half[1:, :],
tran_half[1:, :])
numpy.logical_or(mask[:, 1:] != mask[:, :-1], tran_half[:, :-1],
tran_half[:, :-1])
numpy.logical_or(mask[:, :-1] != mask[:, 1:], tran_half[:, 1:],
tran_half[:, 1:])
sdf[tran_half] = 0.5
# Do all 8 directions of sweep iterativly until distances stop getting smaller...
stop = False
while not stop:
stop = True
code = start_cpp() + """
float sqrt2 = sqrt(2.0);
// Forwards pass...
for (int y=0; y<Nsdf[0]; y++)
{
for (int x=0; x<Nsdf[1]; x++)
{
bool negx = x!=0;
bool negy = y!=0;
if ((negx)&&((SDF2(y, x-1)+1.0)<SDF2(y, x)))
{
SDF2(y, x) = SDF2(y, x-1) + 1.0;
stop = false;
}
if ((negy)&&((SDF2(y-1, x)+1.0)<SDF2(y, x)))
{
SDF2(y, x) = SDF2(y-1, x) + 1.0;
stop = false;
}
if ((negx)&&(negy)&&((SDF2(y-1, x-1)+sqrt2)<SDF2(y, x)))
{
SDF2(y, x) = SDF2(y-1, x-1) + sqrt2;
stop = false;
}
}
}
// Backwards pass...
for (int y=Nsdf[0]-1; y>=0; y--)
{
for (int x=Nsdf[1]-1; x>=0; x--)
{
bool posx = (x+1)!=Nsdf[1];
bool posy = (y+1)!=Nsdf[0];
if ((posx)&&((SDF2(y, x+1)+1.0)<SDF2(y, x)))
{
SDF2(y, x) = SDF2(y, x+1) + 1.0;
stop = false;
}
if ((posy)&&((SDF2(y+1, x)+1.0)<SDF2(y, x)))
{
SDF2(y, x) = SDF2(y+1, x) + 1.0;
stop = false;
}
if ((posx)&&(posy)&&((SDF2(y+1, x+1)+sqrt2)<SDF2(y, x)))
{
SDF2(y, x) = SDF2(y+1, x+1) + sqrt2;
stop = false;
}
}
}
"""
weave.inline(code, ['sdf', 'stop'])
# Add in the sign - negate all pixels that are within the mask, and return...
sdf[mask] *= -1.0
return sdf
def codeC(self, name, escl):
cStats = start_cpp() + """
void %(name)s_stats(PyObject * data, Exemplar * index, void *& out, size_t & outLen)
{
// Make sure the output it at least as large as classCount, and zero it out...
if (outLen<(sizeof(float)*%(classCount)i))
{
outLen = sizeof(float) * %(classCount)i;
out = realloc(out, outLen);
}
for (int i=0; i<(outLen/sizeof(float)); i++)
{
((float*)out)[i] = 0.0;
}
// Iterate and play weighted histogram, growing out as needed...
%(channelType)s cData = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int maxSeen = %(classCount)i;
while (index)
{
int cls = %(channelName)s_get(cData, index->index, 0);
int cap = cls+1;
if (cap>maxSeen) maxSeen = cap;
if ((cap*sizeof(float))>outLen)
{
int zero_start = outLen / sizeof(float);
outLen = cap*sizeof(float);
out = realloc(out, outLen);
for (int i=zero_start; i<cap; i++)
{
((float*)out)[i] = 0.0;
}
}
((float*)out)[cls] += index->weight;
index = index->next;
}
// Correct the output size if needed (It could be too large)...
outLen = maxSeen * sizeof(float);
}
"""%{'name':name, 'channel':self.channel, 'channelName':escl[self.channel]['name'], 'channelType':escl[self.channel]['itype'], 'classCount':self.classCount if self.classCount!=None else 1}
cUpdateStats = start_cpp() + """
void %(name)s_updateStats(PyObject * data, Exemplar * index, void *& inout, size_t & inoutLen)
{
// Iterate and play weighted histogram, growing out as needed...
%(channelType)s cData = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int maxSeen = inoutLen / sizeof(float);
while (index)
{
int cls = %(channelName)s_get(cData, index->index, 0);
int cap = cls+1;
if (cap>maxSeen) maxSeen = cap;
if ((cap*sizeof(float))>inoutLen)
{
int zero_start = inoutLen / sizeof(float);
inoutLen = cap*sizeof(float);
inout = realloc(inout, inoutLen);
for (int i=zero_start; i<cap; i++)
{
((float*)inout)[i] = 0.0;
}
}
((float*)inout)[cls] += index->weight;
index = index->next;
}
}
"""%{'name':name, 'channel':self.channel, 'channelName':escl[self.channel]['name'], 'channelType':escl[self.channel]['itype']}
cEntropy = start_cpp() + """
float %(name)s_entropy(void * stats, size_t statsLen)
{
float sum = 0.0;
int length = statsLen>>2;
for (int i=0; i<length; i++)
{
sum += ((float*)stats)[i];
}
float ret = 0.0;
for (int i=0; i<length; i++)
{
float val = ((float*)stats)[i];
if (val>1e-6)
{
val /= sum;
ret -= val * log(val);
}
}
return ret;
}
"""%{'name':name}
cSummary = start_cpp() + """
void %(name)s_summary(PyObject * data, Exemplar * index, void *& out, size_t & outLen)
{
// Make sure the output it at least as large as classCount, and zero it out...
if (outLen<(sizeof(float)*%(classCount)i))
{
outLen = sizeof(float) * %(classCount)i;
out = realloc(out, outLen);
}
for (int i=0; i<(outLen/sizeof(float)); i++)
{
((float*)out)[i] = 0.0;
}
// Iterate and play weighted histogram, growing out as needed...
%(channelType)s cData = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int maxSeen = %(classCount)i;
while (index)
{
int cls = %(channelName)s_get(cData, index->index, 0);
int cap = cls+1;
if (cap>maxSeen) maxSeen = cap;
if ((cap*sizeof(float))>outLen)
{
int zero_start = outLen / sizeof(float);
outLen = cap*sizeof(float);
out = realloc(out, outLen);
for (int i=zero_start; i<cap; i++)
{
((float*)out)[i] = 0.0;
}
}
((float*)out)[cls] += index->weight;
index = index->next;
}
// Correct the output size if needed (It could be too large)...
outLen = maxSeen * sizeof(float);
}
"""%{'name':name, 'channel':self.channel, 'channelName':escl[self.channel]['name'], 'channelType':escl[self.channel]['itype'], 'classCount':self.classCount if self.classCount!=None else 1}
cUpdateSummary = start_cpp() + """
void %(name)s_updateSummary(PyObject * data, Exemplar * index, void *& inout, size_t & inoutLen)
{
// Iterate and play weighted histogram, growing out as needed...
%(channelType)s cData = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int maxSeen = inoutLen / sizeof(float);
while (index)
{
int cls = %(channelName)s_get(cData, index->index, 0);
int cap = cls+1;
if (cap>maxSeen) maxSeen = cap;
if ((cap*sizeof(float))>inoutLen)
{
int zero_start = inoutLen / sizeof(float);
inoutLen = cap*sizeof(float);
inout = realloc(inout, inoutLen);
for (int i=zero_start; i<cap; i++)
{
((float*)inout)[i] = 0.0;
}
}
((float*)inout)[cls] += index->weight;
index = index->next;
}
}
"""%{'name':name, 'channel':self.channel, 'channelName':escl[self.channel]['name'], 'channelType':escl[self.channel]['itype']}
cError = start_cpp() + """
void %(name)s_error(void * stats, size_t statsLen, void * summary, size_t summaryLen, float & error, float & weight)
{
// Sum the stuff in stats...
int statsSize = statsLen / sizeof(float);
float statsSum = 0.0;
for (int i=0; i<statsSize; i++) statsSum += ((float*)stats)[i];
// Go through and factor in each class from the summary in turn, using an incrimental mean...
int summarySize = summaryLen / sizeof(float);
for (int c=0; c<summarySize; c++)
{
float avgErr = (c<statsSize)?(1.0 - ((float*)stats)[c]/statsSum):1.0;
float w = ((float*)summary)[c];
weight += w;
if (weight>1e-3)
{
error += (avgErr-error) * w/weight;
}
}
}
"""%{'name':name}
return {'stats':cStats, 'updateStats':cUpdateStats, 'entropy':cEntropy, 'summary':cSummary, 'updateSummary':cUpdateSummary, 'error':cError}
def test_compile(self): code = start_cpp(linked_list_gc) + """ """ weave.inline(code, support_code=linked_list_gc)
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from utils.start_cpp import start_cpp
conc_code = start_cpp() + """
// This funky little function is used to resample the concentration parameter of a Dirichlet process, using the previous parameter - allows this parameter to be Gibbs sampled. Also works for any level of a HDP, due to the limited interactions.
// Parameters are:
// pcp - previous concentration parameter.
// n - number of samples taken from the Dirichlet process
// k - number of discretly different samples, i.e. table count in the Chinese restaurant process.
// prior_alpha - alpha value of the Gamma prior on the concentration parameter.
// prior_beta - beta value of the Gamma prior on the concentration parameter.
double sample_dirichlet_proc_conc(double pcp, double n, double k, double prior_alpha = 1.01, double prior_beta = 0.01)
{
if ((n<(1.0-1e-6))||(k<(2.0-1e-6)))
{
return pcp; // Doesn't work in this case, so just repeat.
}
def error(self, goal, gen, es, index = slice(None), weights = None, inc = False, store = None, code = None):
"""Once a tree is trained this method allows you to determine how good it is, using a test set, which would typically be its out-of-bag (oob) test set. Given a test set, possibly weighted, it will return its error rate, as defined by the goal. goal is the Goal object used for trainning, gen the Generator. Also supports incrimental testing, where the information gleened from the test set is stored such that new test exemplars can be added. This is the inc variable - True to store this (potentially large) quantity of information, and update it if it already exists, False to not store it and therefore disallow incrimental learning whilst saving memory. Note that the error rate will change by adding more training data as well as more testing data - you can call it with es==None to get an error score without adding more testing exemplars, assuming it has previously been called with inc==True. store is for internal use only. code can be provided by the relevent parameter, as generated by the errorC method, allowing a dramatic speedup."""
if code!=None and weave!=None:
init = start_cpp(code) + """
float err = 0.0;
float weight = 0.0;
if (dummy==0) // To allow for a dummy run.
{
if (Nindex[0]!=0)
{
Exemplar * test_set = (Exemplar*)malloc(sizeof(Exemplar)*Nindex[0]);
for (int i=0; i<Nindex[0]; i++)
{
int ind = index[i];
test_set[i].index = ind;
test_set[i].weight = weights[ind];
test_set[i].next = &test_set[i+1];
}
test_set[Nindex[0]-1].next = 0;
error(self, data, test_set, err, weight, incNum==1);
free(test_set);
}
else
{
error(self, data, 0, err, weight, incNum==1);
}
}
return_val = err;
"""
data = es.tupleInputC()
dummy = 1 if self==None else 0
incNum = 1 if inc else 0
if weights==None: weights = numpy.ones(es.exemplars(), dtype=numpy.float32)
return weave.inline(init, ['self', 'data', 'index', 'weights', 'incNum', 'dummy'], support_code=code)
else:
# Book-keeping - work out if we need to return a score; make sure there is a store list...
ret = store==None
if ret:
store = []
if isinstance(index, slice): index = numpy.arange(*index.indices(es.exemplars()))
# Update the summary at this node if needed...
summary = None
if es!=None and index.shape[0]!=0:
if self.summary==None: summary = goal.summary(es, index, weights)
else: summary = goal.updateSummary(self.summary, es, index, weights)
if inc: self.summary = summary
# Either recurse to the leafs or include this leaf...
if self.test==None:
# A leaf...
if summary!=None: store.append(goal.error(self.stats, summary))
else:
# Not a leaf...
if es!=None:
res = gen.do(self.test, es, index)
tIndex = index[res==True]
fIndex = index[res==False]
if tIndex.shape[0]!=0: self.true.error(goal, gen, es, tIndex, weights, inc, store)
elif inc==True: self.true.error(goal, gen, None, tIndex, weights, inc, store)
if fIndex.shape[0]!=0: self.false.error(goal, gen, es, fIndex, weights, inc, store)
elif inc==True: self.false.error(goal, gen, None, fIndex, weights, inc, store)
else:
self.true.error(goal, gen, es, index, weights, inc, store)
self.false.error(goal, gen, es, index, weights, inc, store)
# Calculate the weighted average of all the leafs all at once, to avoid an inefficient incrimental calculation, or just sum them up if a weight of None has been provided at any point...
if ret and len(store)!=0:
if None in map(lambda t: t[1], store):
return sum(map(lambda t: t[0], store))
else:
store = numpy.asarray(store, dtype=numpy.float32)
return numpy.average(store[:,0], weights=store[:,1])
def genCodeC(self, name, exemplar_list):
code = start_cpp() + """
struct State%(name)s
{
void * test;
size_t length;
float * dirs; // Vectors giving points uniformly distributed on the hyper-sphere.
int * feat; // The features to index at this moment.
float mean;
float sd;
// Control counters - all count down...
int featRemain;
int dirRemain;
int splitRemain;
};
void %(name)s_init(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
assert(sizeof(int)==4);
// Output state...
state.length = sizeof(int) * %(dims)i + sizeof(float) * (%(dims)i+1);
state.test = malloc(state.length);
// Generate a bunch of random directions...
state.dirs = (float*)malloc(sizeof(float)*%(dims)i*%(dirCount)i);
for (int d=0;d<%(dirCount)i;d++)
{
float length = 0.0;
int base = %(dims)i * d;
for (int f=0; f<%(dims)i; f++)
{
double u = 1.0-drand48();
double v = 1.0-drand48();
float bg = sqrt(-2.0*log(u)) * cos(2.0*M_PI*v);
length += bg*bg;
state.dirs[base+f] = bg;
}
length = sqrt(length);
for (int f=0; f<%(dims)i; f++)
{
state.dirs[base+f] /= length;
}
}
// Which features are currently being used...
state.feat = (int*)malloc(sizeof(int)*%(dims)i);
// Setup the counters so we do the required work when next is called...
state.featRemain = %(dimCount)i;
state.dirRemain = 0;
state.splitRemain = 0;
// Safety...
%(channelType)s cd = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
int featCount = %(channelName)s_features(cd);
if (%(dims)i>featCount)
{
state.featRemain = 0; // Effectivly cancels work.
}
}
bool %(name)s_next(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
if (state.splitRemain==0)
{
%(channelType)s cd = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
if (state.dirRemain==0)
{
if (state.featRemain==0)
{
free(state.feat);
free(state.dirs);
free(state.test);
return false;
}
state.featRemain--;
// Select a new set of features...
int featCount = %(channelName)s_features(cd);
for (int f=0; f<%(dims)i; f++)
{
state.feat[f] = lrand48() %% (featCount-f);
for (int j=0; j<f; j++)
{
if (state.feat[j]<=state.feat[f]) state.feat[f]++;
}
}
// Reset the counter...
state.dirRemain = %(dirCount)i;
}
state.dirRemain--;
// For the new direction calculate the mean and standard deviation with the current features...
float sum = 0.0;
float mean = 0.0;
float mean2 = 0.0;
while (test_set)
{
float x = 0.0;
int base = %(dims)i * state.dirRemain;
for (int f=0; f<%(dims)i; f++)
{
x += state.dirs[base+f] * %(channelName)s_get(cd, test_set->index, state.feat[f]);
}
if (%(ignoreWeights)s)
{
sum += 1.0;
float delta = x - mean;
mean += delta/sum;
mean2 += delta * (x - mean);
}
else
{
float newSum = sum + test_set->weight;
float delta = x - mean;
float mean_delta = delta * test_set->weight / newSum;
mean += mean_delta;
mean2 += sum * delta * mean_delta;
sum = newSum;
}
test_set = test_set->next;
}
state.mean = mean;
state.sd = sqrt(mean2/sum);
if (state.sd<1e-6) state.sd = 1e-6;
// Reset the counter...
state.splitRemain = %(splitCount)i;
}
state.splitRemain--;
// Use the mean and standard deviation to select a split point...
double u = 1.0-drand48();
double v = 1.0-drand48();
float bg = sqrt(-2.0*log(u)) * cos(2.0*M_PI*v);
float split = state.mean + state.sd * bg;
// Store it all in the output...
for (int i=0; i<%(dims)i;i++)
{
((int*)state.test)[i] = state.feat[i];
}
int base = %(dims)i * state.dirRemain;
for (int i=0; i<%(dims)i;i++)
{
((float*)state.test)[%(dims)i+i] = state.dirs[base+i];
}
((float*)state.test)[2*%(dims)i] = split;
return true;
}
""" % {
'name': name,
'channel': self.channel,
'channelName': exemplar_list[self.channel]['name'],
'channelType': exemplar_list[self.channel]['itype'],
'dims': self.dims,
'dimCount': self.dimCount,
'dirCount': self.dirCount,
'splitCount': self.splitCount,
'ignoreWeights': ('true' if self.ignoreWeights else 'false')
}
return (code, 'State' + name)
def evaluateC(gen, es, esclName = 'es'):
"""For a given generator and exemplar set this returns the C code (Actually the support code.) that evaluate can use to accelerate its run time, or None if the various components involved do not support C code generation."""
# First do accessors for the data set...
try:
escl = es.listCodeC(esclName)
except NotImplementedError: return None
code = ''
for channel in escl:
code += channel['get'] + '\n'
code += channel['exemplars'] + '\n'
code += channel['features'] + '\n'
# Now throw in the test code...
try:
code += gen.testCodeC('do_test', escl) + '\n'
except NotImplementedError: return None
# Finally add in the code that recurses through and evaluates the nodes on the provided data...
code += start_cpp() + """
// So we can use an inplace modified linkied list to avoid malloc's during the real work (Weight is included because this code is reused by the generator system, which needs it.)...
struct Exemplar
{
int index;
float weight;
Exemplar * next;
};
// Recursivly does the work...
// node - node of the tree; for an external user this will always be the root.
// data - python tuple containing the inputs needed at each stage.
// test_set - Linked list of entities to analyse.
// out - python list in which the output is to be stored.
void evaluate(PyObject * node, PyObject * data, Exemplar * test_set, PyObject * out)
{
PyObject * test = PyObject_GetAttrString(node, "test");
if (test==Py_None)
{
// Leaf node - assign the relevent stats to the members of the test-set...
PyObject * stats = PyObject_GetAttrString(node, "stats");
while (test_set)
{
Py_INCREF(stats);
PyList_SetItem(out, test_set->index, stats);
test_set = test_set->next;
}
Py_DECREF(stats);
}
else
{
// Branch node - use the test to split the test_set and recurse...
// Tests...
Exemplar * pass = 0;
Exemplar * fail = 0;
void * test_ptr = PyString_AsString(test);
size_t test_len = PyString_Size(test);
while (test_set)
{
Exemplar * next = test_set->next;
if (do_test(data, test_ptr, test_len, test_set->index))
{
test_set->next = pass;
pass = test_set;
}
else
{
test_set->next = fail;
fail = test_set;
}
test_set = next;
}
// Recurse...
if (pass)
{
PyObject * child = PyObject_GetAttrString(node, "true");
evaluate(child, data, pass, out);
Py_DECREF(child);
}
if (fail)
{
PyObject * child = PyObject_GetAttrString(node, "false");
evaluate(child, data, fail, out);
Py_DECREF(child);
}
}
Py_DECREF(test);
}
"""
return code
def give_birth(self, goal, gen, pruner, es, index = slice(None), weights = None, depth = 0, entropy = None, code = None):
"""This recursivly grows the tree until the pruner says to stop. goal is a Goal object, so it knows what to optimise, gen a Generator object that provides tests for it to choose between and pruner is a Pruner object that decides when to stop growing. The exemplar set to train on is then provided, optionally with the indices of which members to use and weights to assign to them (weights align with the exemplar set, not with the relative exemplar indices defined by index. depth is the depth of this node - part of the recursive construction and used by the pruner as a possible reason to stop growing. entropy should match up with self.stats. The static method initC can be called to generate code that can be used to accelerate test selection, but only if it is passed in."""
if entropy==None: entropy = goal.entropy(self.stats)
# Select the best test...
if isinstance(code, str) and weave!=None:
# Do things in C...
init = start_cpp(code) + """
if (Nindex[0]!=0)
{
srand48(rand);
// Create the Exemplar data structure, in triplicate!..
Exemplar * items = (Exemplar*)malloc(sizeof(Exemplar)*Nindex[0]);
Exemplar * splitItems = (Exemplar*)malloc(sizeof(Exemplar)*Nindex[0]);
Exemplar * temp = (Exemplar*)malloc(sizeof(Exemplar)*Nindex[0]);
for (int i=0; i<Nindex[0]; i++)
{
int ind = index[i];
float we = weights[ind];
items[i].index = ind;
items[i].weight = we;
items[i].next = &items[i+1];
splitItems[i].index = ind;
splitItems[i].weight = we;
splitItems[i].next = &splitItems[i+1];
temp[i].next = &temp[i+1];
}
items[Nindex[0]-1].next = 0;
splitItems[Nindex[0]-1].next = 0;
temp[Nindex[0]-1].next = 0;
// Do the work...
selectTest(out, data, items, splitItems, temp, entropy);
// Clean up...
free(temp);
free(splitItems);
free(items);
}
"""
data = es.tupleInputC()
out = dict()
rand = numpy.random.randint(-1000000000,1000000000)
if weights==None: weights = numpy.ones(es.exemplars(), dtype=numpy.float32)
weave.inline(init, ['out', 'data', 'index', 'weights', 'entropy', 'rand'], support_code=code)
if index.shape[0]==0: return
bestTest = out['bestTest']
if bestTest!=None:
bestInfoGain = out['bestInfoGain']
trueStats = out['trueStats']
trueEntropy = out['trueEntropy']
trueIndex = out['trueIndex']
falseStats = out['falseStats']
falseEntropy = out['falseEntropy']
falseIndex = out['falseIndex']
trueIndex.sort() # Not needed to work - to improve cache coherance.
falseIndex.sort() # "
else:
if index.shape[0]==0: return
# Do things in python...
## Details of best test found so far...
bestInfoGain = -1.0
bestTest = None
trueStats = None
trueEntropy = None
trueIndex = None
falseStats = None
falseEntropy = None
falseIndex = None
## Get a bunch of tests and evaluate them against the goal...
for test in gen.itertests(es, index, weights):
# Apply the test, work out which items pass and which fail..
res = gen.do(test, es, index)
tIndex = index[res==True]
fIndex = index[res==False]
# Check its safe to continue...
if tIndex.shape[0]==0 or fIndex.shape[0]==0: continue
# Calculate the statistics...
tStats = goal.stats(es, tIndex, weights)
fStats = goal.stats(es, fIndex, weights)
# Calculate the information gain...
tEntropy = goal.entropy(tStats)
fEntropy = goal.entropy(fStats)
if weights==None:
tWeight = float(tIndex.shape[0])
fWeight = float(fIndex.shape[0])
else:
tWeight = weights[tIndex].sum()
fWeight = weights[fIndex].sum()
div = tWeight + fWeight
tWeight /= div
fWeight /= div
infoGain = entropy - tWeight*tEntropy - fWeight*fEntropy
# Store if the best so far...
if infoGain>bestInfoGain:
bestInfoGain = infoGain
bestTest = test
trueStats = tStats
trueEntropy = tEntropy
trueIndex = tIndex
falseStats = fStats
falseEntropy = fEntropy
falseIndex = fIndex
# Use the pruner to decide if we should split or not, and if so do it...
self.test = bestTest
if bestTest!=None and pruner.keep(depth, trueIndex.shape[0], falseIndex.shape[0], bestInfoGain, self)==True:
# We are splitting - time to recurse...
self.true = Node(goal, gen, pruner, es, trueIndex, weights, depth+1, trueStats, trueEntropy, code)
self.false = Node(goal, gen, pruner, es, falseIndex, weights, depth+1, falseStats, falseEntropy, code)
else:
self.test = None
self.true = None
self.false = None
def initC(goal, gen, es):
# Get the evaluateC code, which this code is dependent on...
code = Node.evaluateC(gen, es, 'es')
if code==None: return None
# Add in the generator code...
escl = es.listCodeC('es')
try:
gCode, gState = gen.genCodeC('gen', escl)
except NotImplementedError:
return None
code += gCode
# Add in the goal code...
try:
gDic = goal.codeC('goal', escl)
except NotImplementedError:
return None
try:
code += gDic['stats']
code += gDic['entropy']
except KeyError:
return None
# And finally add in the code we need to specifically handle the selection of a test for a node...
code += start_cpp() + """
// out - A dictionary to output into; data - The list of entities that represent the exemplar set; items - The set of items to optimise the test for, splitItems - A copy of items, which will be screwed with; temp - Like items, same size, so we can keep a temporary copy; entropy - The entropy of the set of items.
void selectTest(PyObject * out, PyObject * data, Exemplar * items, Exemplar * splitItems, Exemplar * temp, float entropy)
{
// Setup the generator...
%(gState)s state;
gen_init(state, data, items);
// Loop the tests, scoring each one and keeping the best so far...
void * bestTest = 0;
size_t bestTestLen = 0;
void * bestPassStats = 0;
size_t bestPassStatsLen = 0;
float bestPassEntropy = -1.0;
Exemplar * bestPassItems = temp;
int bestPassItemsLen = 0;
void * bestFailStats = 0;
size_t bestFailStatsLen = 0;
float bestFailEntropy = -1.0;
Exemplar * bestFailItems = 0;
int bestFailItemsLen = 0;
float bestGain = 0.0;
Exemplar * pass = splitItems;
void * passStats = 0;
size_t passStatsLength = 0;
Exemplar * fail = 0;
void * failStats = 0;
size_t failStatsLength = 0;
while (gen_next(state, data, items))
{
// Apply the test...
Exemplar * newPass = 0;
float passWeight = 0.0;
Exemplar * newFail = 0;
float failWeight = 0.0;
while (pass)
{
Exemplar * next = pass->next;
if (do_test(data, state.test, state.length, pass->index))
{
pass->next = newPass;
newPass = pass;
passWeight += pass->weight;
}
else
{
pass->next = newFail;
newFail = pass;
failWeight += pass->weight;
}
pass = next;
}
while (fail)
{
Exemplar * next = fail->next;
if (do_test(data, state.test, state.length, fail->index))
{
fail->next = newPass;
newPass = fail;
passWeight += fail->weight;
}
else
{
fail->next = newFail;
newFail = fail;
failWeight += fail->weight;
}
fail = next;
}
pass = newPass;
fail = newFail;
if ((pass==0)||(fail==0))
{
// All data has gone one way - this scernario can not provide an advantage so ignore it.
continue;
}
// Generate the stats objects and entropy...
goal_stats(data, pass, passStats, passStatsLength);
goal_stats(data, fail, failStats, failStatsLength);
float passEntropy = goal_entropy(passStats, passStatsLength);
float failEntropy = goal_entropy(failStats, failStatsLength);
// Calculate the information gain...
float div = passWeight + failWeight;
passWeight /= div;
failWeight /= div;
float gain = entropy - passWeight*passEntropy - failWeight*failEntropy;
// If it is the largest store its output for future consumption...
if (gain>bestGain)
{
bestTestLen = state.length;
bestTest = realloc(bestTest, bestTestLen);
memcpy(bestTest, state.test, bestTestLen);
bestPassStatsLen = passStatsLength;
bestPassStats = realloc(bestPassStats, bestPassStatsLen);
memcpy(bestPassStats, passStats, bestPassStatsLen);
bestFailStatsLen = failStatsLength;
bestFailStats = realloc(bestFailStats, bestFailStatsLen);
memcpy(bestFailStats, failStats, bestFailStatsLen);
bestPassEntropy = passEntropy;
bestFailEntropy = failEntropy;
bestGain = gain;
Exemplar * storeA = bestPassItems;
Exemplar * storeB = bestFailItems;
bestPassItems = 0;
bestPassItemsLen = 0;
bestFailItems = 0;
bestFailItemsLen = 0;
Exemplar * targPass = pass;
while (targPass)
{
// Get an output node...
Exemplar * out;
if (storeA)
{
out = storeA;
storeA = storeA->next;
}
else
{
out = storeB;
storeB = storeB->next;
}
// Store it...
out->next = bestPassItems;
bestPassItems = out;
bestPassItemsLen++;
out->index = targPass->index;
targPass = targPass->next;
}
Exemplar * targFail = fail;
while (targFail)
{
// Get an output node...
Exemplar * out;
if (storeA)
{
out = storeA;
storeA = storeA->next;
}
else
{
out = storeB;
storeB = storeB->next;
}
// Store it...
out->next = bestFailItems;
bestFailItems = out;
bestFailItemsLen++;
out->index = targFail->index;
targFail = targFail->next;
}
}
}
// Output the best into the provided dictionary - quite a lot of information...
if (bestTest!=0)
{
PyObject * t = PyFloat_FromDouble(bestGain);
PyDict_SetItemString(out, "bestInfoGain", t);
Py_DECREF(t);
t = PyString_FromStringAndSize((char*)bestTest, bestTestLen);
PyDict_SetItemString(out, "bestTest", t);
Py_DECREF(t);
t = PyString_FromStringAndSize((char*)bestPassStats, bestPassStatsLen);
PyDict_SetItemString(out, "trueStats", t);
Py_DECREF(t);
t = PyFloat_FromDouble(bestPassEntropy);
PyDict_SetItemString(out, "trueEntropy", t);
Py_DECREF(t);
PyArrayObject * ta = (PyArrayObject*)PyArray_FromDims(1, &bestPassItemsLen, NPY_INT32);
int i = 0;
while (bestPassItems)
{
*(int*)(ta->data + ta->strides[0]*i) = bestPassItems->index;
i++;
bestPassItems = bestPassItems->next;
}
PyDict_SetItemString(out, "trueIndex", (PyObject*)ta);
Py_DECREF(ta);
t = PyString_FromStringAndSize((char*)bestFailStats, bestFailStatsLen);
PyDict_SetItemString(out, "falseStats", t);
Py_DECREF(t);
t = PyFloat_FromDouble(bestFailEntropy);
PyDict_SetItemString(out, "falseEntropy", t);
Py_DECREF(t);
ta = (PyArrayObject*)PyArray_FromDims(1, &bestFailItemsLen, NPY_INT32);
i = 0;
while (bestFailItems)
{
*(int*)(ta->data + ta->strides[0]*i) = bestFailItems->index;
i++;
bestFailItems = bestFailItems->next;
}
PyDict_SetItemString(out, "falseIndex", (PyObject*)ta);
Py_DECREF(ta);
}
else
{
PyDict_SetItemString(out, "bestTest", Py_None);
Py_INCREF(Py_None);
}
// Clean up...
free(bestTest);
free(bestPassStats);
free(bestFailStats);
free(passStats);
free(failStats);
}
"""%{'gState':gState}
return code
def __do_median(self, mask):
"""Applys a median filter to self.uv - pretty simple really. Areas outside the mask are ignored."""
code = start_cpp() + """
int size = radius*2 + 1;
// Iterate and calculate the median for each pixel, writting the output into temp...
for (int y=0;y<Nmask[0];y++)
{
for (int x=0;x<Nmask[1];x++)
{
if (MASK2(y,x)!=0)
{
// Get ranges, bound checked...
int startV = y - radius;
int endV = y + radius;
int startU = x - radius;
int endU = x + radius;
if (startV<0) startV = 0;
if (endV>=Nmask[0]) endV = Nmask[0]-1;
if (startU<0) startU = 0;
if (endU>=Nmask[1]) endU = Nmask[1]-1;
// Zero out the window, so the distances may be summed in...
for (int v=startV;v<=endV;v++)
{
int wv = v - startV;
for (int u=startU;u<=endU;u++)
{
int wu = u - startU;
WIN2(wv,wu) = 0.0;
}
}
// Calculate the distances for each entry - take care to avoid duplicate calculation, even though it makes for some messy code...
for (int v=startV;v<=endV;v++)
{
int wv = v - startV;
for (int u=startU;u<=endU;u++)
{
int wu = u - startU;
if (MASK2(v,u)!=0)
{
int ov = v;
int ou = u;
while (true)
{
ou += 1;
if (ou>endU)
{
ou = startU;
ov += 1;
if (ov>endV) break;
}
if (MASK2(ov,ou)==0) continue;
int wov = ov - startV;
int wou = ou - startU;
float deltaV = UV3(ov,ou,0) - UV3(v,u,0);
float deltaU = UV3(ov,ou,1) - UV3(v,u,1);
float dist = sqrt(deltaU*deltaU + deltaV*deltaV);
WIN2(wv,wu) += dist;
WIN2(wov,wou) += dist;
}
}
}
}
// Find and select the best entry...
float best = 1e100;
for (int v=startV;v<=endV;v++)
{
int wv = v - startV;
for (int u=startU;u<=endU;u++)
{
int wu = u - startU;
if (MASK2(v,u)!=0)
{
if (WIN2(wv,wu)<best)
{
best = WIN2(wv,wu);
TEMP3(y,x,0) = UV3(v,u,0);
TEMP3(y,x,1) = UV3(v,u,1);
}
}
}
}
}
}
}
// Copy from temp into image...
for (int y=0;y<Nmask[0];y++)
{
for (int x=0;x<Nmask[1];x++)
{
if (MASK2(y,x)!=0)
{
UV3(y,x,0) = TEMP3(y,x,0);
UV3(y,x,1) = TEMP3(y,x,1);
}
}
}
"""
uv = self.uv
temp = self.image
win = self.window
radius = self.radiusMF
weave.inline(code, ['mask', 'uv', 'temp', 'win', 'radius'])
def __do_iter(self, iFrom, iTo, mask):
"""Given a from image and a to image this does a single LK iteration, starting from and updating the values in self.uv. If images are smaller than self.uv then it just uses the corner - means same uv object can be used throughout pyramid construction."""
support_code = start_cpp() + """
// Given a t value in [0,1] calculates the weights of the 4 pixels for a bicubic spline and writes them into out, it also writes into dOut the weights to get the splines differential with respect to t.
void BicubicMult(float t, float out[4], float dOut[4])
{
float t2 = t*t;
float t3 = t2*t;
out[0] = -0.5*t + t2 - 0.5*t3;
out[1] = 1.0 - 2.5*t2 + 1.5*t3;
out[2] = 0.5*t + 2.0*t2 - 1.5*t3;
out[3] = -0.5*t2 + 0.5*t3;
dOut[0] = -0.5 + 2.0*t - 1.5*t2;
dOut[1] = -5.0*t + 4.5*t2;
dOut[2] = 0.5 + 4.0*t - 4.5*t2;
dOut[3] = -t + 1.5*t2;
}
// This does bicubic interpolation of an image, getting both values and differentials, and handling boundary conditions. Input must include the output of calls to BicubicMult for both directions - this encodes the fractional part of the coordinate. The user provides the integer part.
void Bicubic(PyArrayObject * image, int y, float multY[4], float dMultY[4], int x, float multX[4], float dMultX[4], float rgb[3], float rgbDy[3], float rgbDx[3])
{
// Handle coordinates, doing boundary checking - we use repetition at the borders, which makes it a simple matter of coordinate clamping at the boundaries...Y
int coordY[4];
coordY[0] = y-1;
for (int i=1;i<4;i++) coordY[i] = coordY[i-1] + 1;
for (int i=0;i<4;i++)
{
if (coordY[i]>=0) break;
coordY[i] = 0;
}
for (int i=3;i>=0;i--)
{
if (coordY[i]<image->dimensions[0]) break;
coordY[i] = image->dimensions[0]-1;
}
int coordX[4];
coordX[0] = x-1;
for (int i=1;i<4;i++) coordX[i] = coordX[i-1] + 1;
for (int i=0;i<4;i++)
{
if (coordX[i]>=0) break;
coordX[i] = 0;
}
for (int i=3;i>=0;i--)
{
if (coordX[i]<image->dimensions[1]) break;
coordX[i] = image->dimensions[1]-1;
}
// Apply in both dimensions to get value sequences interpolated in both, from which you would typically inteprolate the value in a second step - needed due to calculation of differentials...
float iy[4][3]; // Position, rgb. y=dimension you index with.
float ix[4][3]; // ", but with x.
bzero(iy,sizeof(float)*4*3);
bzero(ix,sizeof(float)*4*3);
for (int v=0;v<4;v++)
{
char * baseV = image->data + coordY[v]*image->strides[0];
for (int u=0;u<4;u++)
{
float * val = (float*)(baseV + coordX[u]*image->strides[1]);
for (int c=0;c<3;c++)
{
iy[v][c] += multX[u] * val[c];
ix[u][c] += multY[v] * val[c];
}
}
}
// Use one dimension and a further step to get the value...
bzero(rgb,sizeof(float)*3);
for (int u=0;u<4;u++)
{
for (int c=0;c<3;c++) rgb[c] += ix[u][c] * multX[u];
}
// Use both dimensions followed by a differential step to get the differentials for both dx and dy...
bzero(rgbDy,sizeof(float)*3);
for (int v=0;v<4;v++)
{
for (int c=0;c<3;c++) rgbDy[c] += iy[v][c] * dMultY[v];
}
bzero(rgbDx,sizeof(float)*3);
for (int u=0;u<4;u++)
{
for (int c=0;c<3;c++) rgbDx[c] += ix[u][c] * dMultX[u];
}
}
"""
code = start_cpp(support_code) + """
// Iterate over the pixels and calculate as estimate for each...
for (int y=0;y<NiFrom[0];y++)
{
for (int x=0;x<NiFrom[1];x++)
{
if (MASK2(y,x)!=0)
{
// Get the range to search - to avoid sampling values outside the image (For the from image - to image is allowed to go outside the range, as handled by the interpolation functions.)...
int yStart = y - radius;
int yEnd = y + radius;
int xStart = x - radius;
int xEnd = x + radius;
if (yStart<0) yStart = 0;
if (yEnd>=NiFrom[0]) yEnd = NiFrom[0] - 1;
if (xStart<0) xStart = 0;
if (xEnd>=NiFrom[1]) xEnd = NiFrom[1] - 1;
// Get the offset from uv, split into integer and fractional parts and calculate the weights for the bicubic interpolation...
int oy = int(UV3(y,x,0));
float ty = UV3(y,x,0) - oy;
float multY[4];
float dMultY[4];
BicubicMult(ty, multY, dMultY);
int ox = int(UV3(y,x,1));
float tx = UV3(y,x,1) - ox;
float multX[4];
float dMultX[4];
BicubicMult(tx, multX, dMultX);
// Calculate the b value and structural tensor, simultaneously, to avoid computing derivatives repeatedly...
float st[3] = {0.0,0.0,0.0}; // Linearised symmetric matrix - [0][0], [0][1]/[1][0], [1][1].
float b[2] = {0.0,0.0};
for (int v=yStart;v<=yEnd;v++)
{
for (int u=xStart;u<=xEnd;u++)
{
if (MASK2(v,u)!=0)
{
// Get the value in the from image...
float * from = (float*)(iFrom_array->data + v*iFrom_array->strides[0] + u*iFrom_array->strides[1]);
// Get the value and differential in the to image...
float rgb[3];
float rgbDy[3];
float rgbDx[3];
Bicubic(iTo_array, v+oy, multY, dMultY, u+ox, multX, dMultX, rgb, rgbDy, rgbDx);
// Loop the colour channels - same calculations for each...
for (int c=0;c<3;c++)
{
// Update the structural tensor...
st[0] += rgbDy[c] * rgbDy[c];
st[1] += rgbDx[c] * rgbDy[c];
st[2] += rgbDx[c] * rgbDx[c];
// Update b...
float diff = from[c] - rgb[c];
b[0] += rgbDy[c] * diff;
b[1] += rgbDx[c] * diff;
}
}
}
}
// Invert the structural tensor, solve the equation, update the uv entry...
double det = double(st[0])*double(st[2]) - double(st[1])*double(st[1]);
if (fabs(det)>1e-9)
{
float temp = st[0];
st[0] = st[2];
st[2] = temp;
st[1] *= -1.0;
st[0] /= det;
st[1] /= det;
st[2] /= det;
// st is now inverted - easy matter to calculate the change...
float dv = st[0]*b[0] + st[1]*b[1];
float du = st[1]*b[0] + st[2]*b[1];
// Only apply the change if it is sensible - approximation is only good for a pixel or so, so ignore if greater than 2 as it being crazy...
float changeSqr = dv*dv + du*du;
if (changeSqr<(2*2))
{
UV3(y,x,0) += dv;
UV3(y,x,1) += du;
}
}
}
}
}
"""
uv = self.uv
radius = self.radiusLK
weave.inline(code, ['iFrom', 'iTo', 'mask', 'uv', 'radius'],
support_code=support_code)
def addTrainC(goal, gen, es, esclName = 'es'):
"""Provides C code that the addTrain method can use to accelerate itself - standard rules about code being unique for each combination of input types applies."""
# First do accessors for the data set...
try:
escl = es.listCodeC(esclName)
except NotImplementedError: return None
code = ''
for channel in escl:
code += channel['get'] + '\n'
code += channel['exemplars'] + '\n'
code += channel['features'] + '\n'
# Throw in the test code...
try:
code += gen.testCodeC('do_test', escl) + '\n'
except NotImplementedError: return None
# Definition of Exemplar...
code += start_cpp() + """
// So we can use an inplace modified linkied list to avoid malloc's during the real work...
struct Exemplar
{
int index;
float weight;
Exemplar * next;
};
"""
# Add the needed goal code...
try:
gDic = goal.codeC('goal', escl)
except NotImplementedError:
return None
try:
code += gDic['updateStats']
except KeyError:
return None
code += start_cpp() + """
void addTrain(PyObject * node, PyObject * data, Exemplar * test_set)
{
// Update the stats at this node...
PyObject * stats = PyObject_GetAttrString(node, "stats");
size_t stLen = PyString_Size(stats);
void * st = malloc(stLen);
memcpy(st, PyString_AsString(stats), stLen);
goal_updateStats(data, test_set, st, stLen);
PyObject * t = PyString_FromStringAndSize((char*)st, stLen);
PyObject_SetAttrString(node, "stats", t);
Py_DECREF(t);
free(st);
Py_DECREF(stats);
// If its not a leaf recurse down and do its children also...
PyObject * test = PyObject_GetAttrString(node, "test");
if (test!=Py_None)
{
// Tests...
Exemplar * pass = 0;
Exemplar * fail = 0;
void * test_ptr = PyString_AsString(test);
size_t test_len = PyString_Size(test);
while (test_set)
{
Exemplar * next = test_set->next;
if (do_test(data, test_ptr, test_len, test_set->index))
{
test_set->next = pass;
pass = test_set;
}
else
{
test_set->next = fail;
fail = test_set;
}
test_set = next;
}
// Recurse...
if (pass!=0)
{
PyObject * child = PyObject_GetAttrString(node, "true");
addTrain(child, data, pass);
Py_DECREF(child);
}
if (fail!=0)
{
PyObject * child = PyObject_GetAttrString(node, "false");
addTrain(child, data, fail);
Py_DECREF(child);
}
}
Py_DECREF(test);
}
"""
return code
def leftRightNegLogProbWord(sample, doc, cluster, particles, cap):
"""Does a left to right estimate of the negative log probability of the words in the given document, given a sample, the documents abnormalities and a cluster assignment. cap defines a cap on the number of documents resampled before each word is sampled for inclusion - set to a negative number for no cap, but be warned that the algorithm is then O(n^2) with regard to the number of words in the document. Should be set quite high in practise for a reasonable trade off between quality and run-time."""
code = start_cpp(shared_code) + """
// Setup - create the state, extract the document, set its cluster...
State state;
StatePyToCpp(stateIn, &state);
Document & doc = state.doc[0];
if (cluster>=0)
{
// Existing cluster...
doc.SetCluster(state.clusters.Index(cluster));
}
else
{
// New cluster...
ItemRef<Cluster,Conc> * newC = state.clusters.Append();
newC->Body().alpha = state.rho.alpha;
newC->Body().beta = state.rho.beta;
newC->Body().conc = state.rho.conc;
float * bmn = new float[state.behCount];
float bmnDiv = 0.0;
for (int b=0;b<state.behCount;b++)
{
bmn[b] = state.phi[b];
bmnDiv += state.phi[b];
}
for (int b=0;b<state.behCount;b++) bmn[b] /= bmnDiv;
newC->SetBMN(bmn);
doc.SetCluster(newC);
}
// If the cap is negative set it to include all words, otherwise we need some storage...
int * samIndex = 0;
if (cap<0) cap = doc.SampleCount();
else
{
samIndex = new int[cap];
}
// Create some memory for storing the results into, zeroed out...
float * samProb = new float[doc.SampleCount()];
for (int s=0;s<doc.SampleCount();s++) samProb[s] = 0.0;
// Do all the particles, summing the results into the samProb array...
for (int p=0;p<particles;p++)
{
// Reset the document to have no assignments to words...
for (int s=0;s<doc.SampleCount();s++)
{
doc.GetSample(s).SetDocInst(0);
}
// Iterate and factor in the result from each sample...
for (int s=0;s<doc.SampleCount();s++)
{
// Resample preceding samples - 3 scenarios with regards to the cap...
// (Note that duplication is allowed in the random sample selection - whilst strictly forbidden the situation is such that it can not cause any issues.)
if (s<=cap)
{
// Less or equal number of samples than the cap - do them all...
for (int s2=0;s2<s;s2++)
{
ResampleSample(state, doc, doc.GetSample(s2));
}
}
else
{
if (s<=cap*2)
{
// Need to miss some samples out, but due to numbers its best to randomly select the ones to miss rather than the ones to do...
int missCount = s-cap;
for (int m=0;m<missCount;m++) samIndex[m] = sample_nat(s);
qsort(samIndex, missCount, sizeof(int), compareInt);
for (int s2=0;s2<samIndex[0];s2++)
{
ResampleSample(state, doc, doc.GetSample(s2));
}
for (int m=0;m<missCount-1;m++)
{
for (int s2=samIndex[m]+1;s2<samIndex[m+1];s2++)
{
ResampleSample(state, doc, doc.GetSample(s2));
}
}
for (int s2=samIndex[missCount-1]+1;s2<s;s2++)
{
ResampleSample(state, doc, doc.GetSample(s2));
}
}
else
{
// Need to select a subset of samples to do...
for (int m=0;m<cap;m++) samIndex[m] = sample_nat(s);
qsort(samIndex, cap, sizeof(int), compareInt);
for (int m=0;m<cap;m++)
{
ResampleSample(state, doc, doc.GetSample(samIndex[m]));
}
}
}
// Calculate the contribution of this sample, whilst simultaneously filling out so we can make a draw from them...
float pSum = CalcSampleProb(state, doc, doc.GetSample(s));
samProb[s] += (pSum - samProb[s]) / (p+1);
// Draw an assignment for the current sample, ready for the next iteration...
ResampleSample(state, doc, doc.GetSample(s), pSum);
}
}
// Sumarise the results buffer into a single log probability and return it...
float ret = 0.0;
for (int s=0;s<doc.SampleCount();s++) ret += log(samProb[s]);
return_val = ret;
// Clean up...
delete[] samIndex;
delete[] samProb;
"""
stateIn = State(doc, Params())
stateIn.setGlobalParams(sample)
stateIn.addPrior(sample)
ret = weave.inline(code,['stateIn','cluster','particles','cap'] , support_code=shared_code)
return -ret # Convert to negative log on the return - before then stick to positive.
def prob(self, sample):
"""Given a sample vector, as something that numpy.asarray can interpret, return the normalised probability of the sample. All values must be correct for this to work. Has inline C, but if that isn't working the implimentation is fully vectorised, so should be quite fast despite being in python."""
global weave
try:
if weave == None: raise Exception()
code = start_cpp() + """
float ret = 0.0;
for (int i=0; i<Nweight[0]; i++)
{
if (weight[i]>1e-6)
{
// Calculate the delta...
for (int j=0; j<Nmean[1]; j++)
{
TEMP2(0, j) = SAMPLE1(j) - MEAN2(i, j);
TEMP2(1, j) = 0.0;
}
// Multiply the precision with the delta and put it into TEMP2(1, ...)...
for (int j=0; j<Nmean[1]; j++)
{
for (int k=0; k<Nmean[1]; k++)
{
TEMP2(1, j) += PREC3(i, j, k) * TEMP2(0, k);
}
}
// Dot product TEMP2(0, ...) and TEMP2(1, ...) to get the core of the distribution...
float core = 0.0;
for (int j=0; j<Nmean[1]; j++)
{
core += TEMP2(0, j) * TEMP2(1, j);
}
// Factor in the rest, add it to the return...
float val = weight[i] * exp(-0.5 * core + log_norm[i]);
if (std::isfinite(val)) ret += val;
}
}
return_val = ret;
"""
sample = numpy.asarray(sample, dtype=numpy.float32)
weight = self.weight
mean = self.mean
prec = self.prec
log_norm = self.log_norm
temp = self.temp
return weave.inline(
code, ['sample', 'weight', 'mean', 'prec', 'log_norm', 'temp'])
except Exception, e:
if weave != None:
print e
weave = None
nzi = numpy.nonzero(self.weight)[0]
sample = numpy.asarray(sample)
delta = numpy.reshape(sample,
(1, self.mean.shape[1])) - self.mean[nzi, :]
nds = (nzi.shape[0], delta.shape[1], 1)
core = (numpy.reshape(delta, nds) *
self.prec[nzi, :, :]).sum(axis=1)
core = (core * delta).sum(axis=1)
core *= -0.5
core += self.log_norm[nzi]
core = numpy.exp(core)
core *= self.weight[nzi]
return core[numpy.isfinite(core)].sum() # Little bit of safety.
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <http://www.gnu.org/licenses/>.
from utils.start_cpp import start_cpp
from utils.numpy_help_cpp import numpy_util_code
from dp_utils.sampling_cpp import sampling_code
smp_code = numpy_util_code + sampling_code + start_cpp() + """
#ifndef SMP_CODE
#define SMP_CODE
class SMP
{
public:
// Basic constructor - after construction before anything else the Init method must be called...
SMP()
:fia(0),priorMN(0),sam(0),samPos(0),samTemp(0),power(0),temp(0)
{}
// Constructor that calls the init method...
SMP(int flagSize, int fliSize)
:fia(0),priorMN(0),sam(0),samPos(0),samTemp(0),power(0),temp(0)
{
def cost(self, cost, sdf, analyse, grad):
"""Given the cost difference (negative log probability) of choosing the pixel belongs to a line over does not belong to a line, a signed distance function and an output into which it will write the gradient of the cost this returns the cost of the curretn sdf, with the gradient of the cost written into grad."""
support = start_cpp() + """
int comp_float(const void * a, const void * b)
{
float fa = *(float*)a;
float fb = *(float*)b;
if (fa<fb) return -1;
if (fb<fa) return 1;
return 0;
}
"""
code = start_cpp() + """
float ret = 0.0;
// Zero out the gradiant output...
for (int y=0; y<Nsdf[0]; y++)
{
for (int x=0; x<Nsdf[1]; x++)
{
GRAD2(y, x) = 0.0;
}
}
// First go through and add in the unary cost for each pixel - we smooth it at the boundary using a sigmoid, as that helps the solver, represents the fact we expect there to be a little noise and keeps it differentiable...
for (int y=0; y<Nsdf[0]; y++)
{
for (int x=0; x<Nsdf[1]; x++)
{
if (ANALYSE2(y, x)==0) continue;
float t = -unary_sharpness * SDF2(y, x);
if (t<-16.0) t = -16.0; // Avoid really extreme values.
if (t>16.0) t = 16.0; // "
float val = 1.0 / (1.0 + exp(-t));
ret += COST2(y, x) * val;
GRAD2(y, x) += -COST2(y, x) * unary_sharpness * val * (1.0-val);
}
}
// Now do the neighbour costs - use a robust mechanism to infer the correct orientation for the line associated (e.g. tangent at nearest point) with each pixel and its distance, and then peanalise deviation from that distance...
static const char dx[8] = {-1, 0, 1, 1, 1, 0, -1, -1};
static const char dy[8] = {-1, -1, -1, 0, 1, 1, 1, 0};
static const float div[8] = {sqrt(2), 1, sqrt(2), 1, sqrt(2), 1, sqrt(2), 1};
const float tgs = tertiary_grad * tertiary_grad;
for (int y=1; y<Nsdf[0]-1; y++)
{
for (int x=1; x<Nsdf[1]-1; x++)
{
// Skip pixels that are not close enough to a line to matter...
if (ANALYSE2(y, x)==0) continue;
// Estimate the 'correct' direction and then distance to the closest line of this pixel, in a super-robust, if expensive, way...
float bestMedian = SDF2(y, x);
float bestMAD = 1e64;
for (int ni=0; ni<8; ni++)
{
// Estimate the perpendicular to the line direction from the 3 neighbours under consideration - a maximum liklihood mean direction of a Fisher distribution (Which for two pixels would be the maximum liklihood fit)...
float nx = 0.0;
float ny = 0.0;
bool oob = false; // Skip set if one of the pixels is not of interest.
for (int oi=0; oi<3; oi++)
{
int i = (ni+oi) % 8;
if (ANALYSE2(y+dy[i], x+dx[i])==0)
{
oob = true;
break;
}
float l = SDF2(y + dy[i], x + dx[i]) - SDF2(y, x);
l /= div[i];
nx += l * dx[i];
ny += l * dy[i];
}
if (oob) continue;
// Normalise the perpendicular to the projection line...
float len = sqrt(nx*nx + ny*ny);
if (len<1e-3) continue;
nx /= len;
ny /= len;
// Use the proposed line to calculate all 8 estimates...
float e[8];
for (int i=0; i<8; i++)
{
float dot = nx * dx[i] + ny * dy[i];
e[i] = SDF2(y + dy[i], x + dx[i]) + dot;
}
// Use the estimates to calculate the median...
qsort(e, 8, sizeof(float), comp_float);
float median = 0.5 * (e[3] + e[4]);
// Mess around and then calculate the MAD...
for (int i=0; i<8; i++)
{
e[i] = fabs(e[i] - median);
}
qsort(e, 8, sizeof(float), comp_float);
float MAD = 0.5 * (e[3] + e[4]);
// If its the best MAD thus far record it...
if (MAD<bestMAD)
{
bestMedian = median;
bestMAD = MAD;
}
}
// Calculate the difference between the robust estimate of the correct distance and the current distance, and use that in a psuedo Huber to update the cost and gradiant...
float err = SDF2(y, x) - bestMedian;
double inner = sqrt(1.0 + err*err/tgs);
ret += tgs * (inner - 1.0);
GRAD2(y, x) += err / inner;
}
}
// Handle the return...
return_val = ret;
"""
unary_sharpness = self.unary_sharpness
tertiary_grad = self.tertiary_grad
return weave.inline(code, [
'cost', 'sdf', 'analyse', 'grad', 'unary_sharpness',
'tertiary_grad'
],
support_code=support)
def genCodeC(self, name, exemplar_list):
code = start_cpp() + """
struct State%(name)s
{
void * test; // Will be the length of a 32 bit int followed by a float.
size_t length;
int dimRemain;
int splitRemain;
int feat;
float mean;
float sd;
};
void %(name)s_init(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
assert(sizeof(int)==4);
state.length = sizeof(int) + sizeof(float);
state.test = malloc(state.length);
state.dimRemain = %(dimCount)i;
state.splitRemain = 0;
}
bool %(name)s_next(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
// If we have used up all splits for this feature select a new one...
if (state.splitRemain==0)
{
// If we have run out of features to select we are done - return as such...
if (state.dimRemain==0)
{
free(state.test);
return false;
}
state.dimRemain--;
// Get the relevent channels object...
%(channelType)s cd = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
// Select a new feature...
state.feat = lrand48() %% %(channelName)s_features(cd);
// Calculate the mean and standard deviation of the data set, for the selected feature...
float sum = 0.0;
float mean = 0.0;
float mean2 = 0.0;
while (test_set)
{
float x = %(channelName)s_get(cd, test_set->index, state.feat);
if (%(ignoreWeights)s)
{
sum += 1.0;
float delta = x - mean;
mean += delta/sum;
mean2 += delta * (x - mean);
}
else
{
float newSum = sum + test_set->weight;
float delta = x - mean;
float mean_delta = delta * test_set->weight / newSum;
mean += mean_delta;
mean2 += sum * delta * mean_delta;
sum = newSum;
}
test_set = test_set->next;
}
state.mean = mean;
state.sd = sqrt(mean2/sum);
if (state.sd<1e-6) state.sd = 1e-6;
state.splitRemain = %(splitCount)i;
}
// Output a split point drawn from the Gaussian...
state.splitRemain--;
double u = 1.0-drand48();
double v = 1.0-drand48();
float bg = sqrt(-2.0*log(u)) * cos(2.0*M_PI*v);
float split = state.mean + state.sd * bg;
((int*)state.test)[0] = state.feat;
((float*)state.test)[1] = split;
return true;
}
""" % {
'name': name,
'channel': self.channel,
'channelName': exemplar_list[self.channel]['name'],
'channelType': exemplar_list[self.channel]['itype'],
'dimCount': self.dimCount,
'splitCount': self.splitCount,
'ignoreWeights': ('true' if self.ignoreWeights else 'false')
}
return (code, 'State' + name)
def answer_batch(self, stats_lists, which, es, indices, trees):
# As this version might be dealing with lots of data we include a scipy.weave based optimisation...
if weave!=None:
code = start_cpp() + """
// Find out what we need to calculate...
bool doProbList = false;
bool doGen = false;
bool doGenList = false;
int wLength = PyList_Size(which);
int * wCodes = (int*)malloc(sizeof(int) * wLength);
for (int i=0; i<wLength; i++)
{
char * s = PyString_AsString(PyList_GetItem(which, i));
wCodes[i] = 0; // prob
if (strcmp(s,"best")==0) wCodes[i] = 1;
if (strcmp(s,"prob_samples")==0) {doProbList = true; wCodes[i] = 2;}
if (strcmp(s,"gen")==0) {doGen = true; wCodes[i] = 3;}
if (strcmp(s,"gen_list")==0) {doGenList = true; wCodes[i] = 4;}
}
// Buffers that are needed...
float * probBuf = 0;
float * genBuf = 0;
// Prep the return value...
int item_count = PyList_Size(stats_lists);
PyObject * ret = PyList_New(item_count);
// Loop through and do each exemplar in turn, adding its result to the return list...
for (int i=0; i<item_count; i++)
{
// Get the list of stats objects...
PyObject * stats = PyList_GetItem(stats_lists, i);
int statCount = PyList_Size(stats);
// Iterate the list and calculate the size of the largest element...
npy_intp vecLength = 0;
for (int j=0; j<statCount; j++)
{
PyObject * s = PyList_GetItem(stats, j);
int len = PyString_Size(s) / sizeof(float);
if (len>vecLength) vecLength = len;
}
// Resize the buffers accordingly, zero them...
probBuf = (float*)realloc(probBuf, sizeof(float)*vecLength);
for (int j=0; j<vecLength; j++) probBuf[j] = 0.0;
if (doGen)
{
genBuf = (float*)realloc(genBuf, sizeof(float)*vecLength);
for (int j=0; j<vecLength; j++) genBuf[j] = 0.0;
}
// Iterate the list and generate the various outputs we need (There are potentially 4 of them.)...
PyObject * probList = 0;
PyObject * genList = 0;
if (doProbList) probList = PyList_New(statCount);
if (doGenList) genList = PyList_New(statCount);
for (int j=0; j<statCount; j++)
{
PyObject * s = PyList_GetItem(stats, j);
int len = PyString_Size(s) / sizeof(float);
float * dist = (float*)(void*)PyString_AsString(s);
float sum = 0.0;
for (int k=0; k<len; k++) sum += dist[k];
if (sum<1e-6) sum = 1e-6; // For safety against divide by zero.
for (int k=0; k<len; k++) probBuf[k] += dist[k] / sum;
if (doProbList)
{
PyObject * arr = PyArray_ZEROS(1, &vecLength, NPY_FLOAT, 0);
for (int k=0; k<len; k++) *(float*)PyArray_GETPTR1(arr, k) = dist[k] / sum;
PyList_SetItem(probList, j, arr);
}
if ((doGen)||(doGenList))
{
PyObject * t = PyList_GetItem(root_stats, j);
float * div = (float*)(void*)PyString_AsString(t);
if (doGen)
{
for (int k=0; k<len; k++) genBuf[k] += dist[k] / div[k];
}
if (doGenList)
{
PyObject * arr = PyArray_ZEROS(1, &vecLength, NPY_FLOAT, 0);
for (int k=0; k<len; k++) *(float*)PyArray_GETPTR1(arr, k) = dist[k] / div[k];
PyList_SetItem(genList, j, arr);
}
}
}
// Normalise the buffers...
{
float sum = 0.0;
for (int j=0; j<vecLength; j++) sum += probBuf[j];
for (int j=0; j<vecLength; j++) probBuf[j] /= sum;
}
if (doGen)
{
for (int j=0; j<vecLength; j++) genBuf[j] /= statCount;
}
// Iterate the proxy for which, and store the required items in the correct positions...
PyObject * ans = PyTuple_New(wLength);
for (int j=0; j<wLength; j++)
{
PyObject * obj = 0;
switch(wCodes[j])
{
case 0: // prob
{
obj = PyArray_EMPTY(1, &vecLength, NPY_FLOAT, 0);
for (int k=0; k<vecLength; k++) *(float*)PyArray_GETPTR1(obj, k) = probBuf[k];
}
break;
case 1: // best
{
int best = 0;
for (int k=1; k<vecLength; k++)
{
if (probBuf[k]>probBuf[best]) best = k;
}
obj = PyInt_FromLong(best);
}
break;
case 2: // prob_samples
{
obj = probList;
Py_INCREF(obj);
}
break;
case 3: // gen
{
obj = PyArray_EMPTY(1, &vecLength, NPY_FLOAT, 0);
for (int k=0; k<vecLength; k++) *(float*)PyArray_GETPTR1(obj, k) = genBuf[k];
}
break;
case 4: // gen_list
{
obj = genList;
Py_INCREF(obj);
}
break;
}
PyTuple_SetItem(ans, j, obj);
}
// Store the answer tuple for this exemplar...
PyList_SetItem(ret, i, ans);
// Some cleaning up...
Py_XDECREF(genList);
Py_XDECREF(probList);
}
// Clean up...
free(probBuf);
free(genBuf);
free(wCodes);
// Return the list of results...
return_val = ret;
Py_XDECREF(ret);
"""
root_stats = map(lambda t: t.stats, trees)
single = isinstance(which, str)
if single: which = [which]
ret = weave.inline(code, ['stats_lists', 'which', 'root_stats'])
if single: ret = map(lambda r: r[0], ret)
return ret
else:
return map(lambda (i, stats_list): self.answer(stats_list, which, es, indices[i], trees), enumerate(stats_lists))
def genCodeC(self, name, exemplar_list):
code = start_cpp() + """
struct State%(name)s
{
void * test; // Will be the length of two 32 bit ints.
size_t length;
int feat; // Current feature.
int featRemain; // How many more times we need to select a feature to play with.
int * value; // List of values drawn from the feature - we return each in turn.
int valueLength; // Reduced as each feature is drawn.
float * weight; // Aligns with value; temporary used for the sampling.
};
int %(name)s_int_comp(const void * lhs, const void * rhs)
{
return (*(int*)lhs) - (*(int*)rhs);
}
void %(name)s_init(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
assert(sizeof(int)==4);
state.test = malloc(2*sizeof(int));
state.length = 2*sizeof(int);
state.feat = -1;
state.featRemain = %(featCount)i;
state.value = (int*)malloc(%(valueCount)i*sizeof(int));
state.valueLength = 0;
state.weight = (float*)malloc(%(valueCount)i*sizeof(float));
}
bool %(name)s_next(State%(name)s & state, PyObject * data, Exemplar * test_set)
{
// Check if we need to create a new set of values...
if (state.valueLength==0)
{
// Check if we are done - if so clean up and return...
if (state.featRemain==0)
{
free(state.weight);
free(state.value);
free(state.test);
return false;
}
state.featRemain--;
// Get the relevent channels object...
%(channelType)s cd = (%(channelType)s)PyTuple_GetItem(data, %(channel)i);
// Select a new feature...
state.feat = lrand48() %% %(channelName)s_features(cd);
// Generate a set of values - use a method based on a single pass through the linked list...
float minWeight = 1.0;
while (test_set)
{
if (test_set->weight>1e-6)
{
float w = pow(drand48(), 1.0/test_set->weight);
if (state.valueLength<%(valueCount)i)
{
state.value[state.valueLength] = test_set->index;
state.weight[state.valueLength] = w;
if (minWeight>w) minWeight = w;
state.valueLength++;
}
else
{
if (w>minWeight) // Below is not very efficient, but don't really care - valueCount tends to be low enough that optimisation is not worthwhile..
{
int lowest = 0;
for (int i=1; i<%(valueCount)i; i++)
{
if (state.weight[lowest]>state.weight[i]) lowest = i;
}
state.value[lowest] = test_set->index;
state.weight[lowest] = w;
minWeight = 1.0;
for (int i=0; i<%(valueCount)i; i++)
{
if (minWeight>state.weight[i]) minWeight = state.weight[i];
}
}
}
}
test_set = test_set->next;
}
// Convert exemplar numbers to actual values...
for (int i=0; i<state.valueLength; i++)
{
state.value[i] = %(channelName)s_get(cd, state.value[i], state.feat);
}
// Remove duplicates...
qsort(state.value, state.valueLength, sizeof(int), %(name)s_int_comp);
int out = 1;
for (int i=1; i<state.valueLength; i++)
{
if (state.value[i]!=state.value[i-1])
{
state.value[out] = state.value[i];
out++;
}
}
state.valueLength = out;
}
// Get and arrange as the output the next value...
state.valueLength -= 1;
((int*)state.test)[0] = state.feat;
((int*)state.test)[1] = state.value[state.valueLength];
return true;
}
""" % {
'name': name,
'channel': self.channel,
'channelName': exemplar_list[self.channel]['name'],
'channelType': exemplar_list[self.channel]['itype'],
'featCount': self.featCount,
'valueCount': self.valueCount
}
return (code, 'State' + name)
def answer_batch(self, stats_lists, which, es, indices, trees):
if weave!=None:
esAccess = es.codeC(0, 'es')
code = start_cpp() + """
// Prepare the access to the es...
%(itype)s es = (%(itype)s)PyList_GetItem(esData, 0);
// Iterate and process each stat list in turn...
int item_count = PyList_Size(stats_lists);
PyObject * ret = PyList_New(item_count);
for (int i=0; i<item_count; i++)
{
// Get the list of stats objects...
PyObject * stats = PyList_GetItem(stats_lists, i);
int statCount = PyList_Size(stats);
// Iterate the list and handle each element in turn...
float p = 0.0;
for (int j=0; j<statCount; j++)
{
// Extract the information regarding the specific stat object...
float * params = (float*)(void*)PyString_AsString(PyList_GetItem(stats, j));
float * mean = params + 3;
float * prec = mean + feats;
// Put the delta into the temporary storage...
for (int k=0; k<feats; k++)
{
TEMP2(0, k) = es_get(es, indices[i], k) - mean[k];
TEMP2(1, k) = 0.0; // Preparation for the next bit.
}
// Calculate the multiplication with the precision...
for (int k=0; k<feats; k++)
{
for (int l=0; l<feats; l++)
{
TEMP2(1, k) += prec[feats*k+l] * TEMP2(0, l);
}
}
float d = 0.0;
for (int k=0; k<feats; k++)
{
d += TEMP2(0, k) * TEMP2(1, k);
}
// Do the final parts required...
p += exp(params[2] - 0.5 * d);
}
p /= statCount;
// Store the calculated probability...
PyObject * ans = PyFloat_FromDouble(p);
PyList_SetItem(ret, i, ans);
}
// Return...
return_val = ret;
Py_XDECREF(ret);
"""%{'itype':esAccess['itype']}
feats = self.feats
esData = [esAccess['input']]
temp = self.temp
ret = weave.inline(code, ['stats_lists', 'indices', 'feats', 'esData', 'temp'], support_code = esAccess['get'])
if isinstance(which, str): return ret
else:
return map(lambda p: tuple([p] * len(which)) , ret)
else:
return map(lambda (i, stats_list): self.answer(stats_list, which, es, indices[i], trees), enumerate(stats_lists))
def test_compile(self):
code = start_cpp(linked_list_gc) + """
"""
weave.inline(code, support_code=linked_list_gc)
def run(self):
response = Gtk.Dialog.run(self)
if response == Gtk.ResponseType.OK and self.let.auto_model != None:
# Clear up all tags/splits currently on the line graph before we start this crazy dance...
start_time = time.clock()
print 'Terminating existing tags/splits...'
for tag in self.let.line.get_tags():
self.let.line.rem(tag[1], tag[2])
for split in self.let.line.get_splits():
self.let.line.rem(split[0], split[1])
self.let.line.segment(
) # Just incase it goes pear shaped - stops errors on returning to the original interface.
# Extract features from the line graph...
print 'Extracting features...'
start = time.time()
feats = self.let.line.features(**self.let.auto_model['feat'])
end = time.time()
print '...done in %.1f seconds' % (end - start)
# Run the model on the features...
print 'Classifying features...'
start = time.time()
probs = self.let.auto_model['model'].predict(feats)[0][
'prob'] # [vertex, class probability]
end = time.time()
print '...done in %.1f seconds' % (end - start)
# Extract the location of each feature in line space...
print 'Getting vertices in line space...'
hg = self.let.ruled.homography
ihg = la.inv(hg)
pos = self.let.line.pos(ihg)
min_x = pos[:, 0].min(
) - 1.0 # Bias term to give space for the tail 'ligaments'.
max_x = pos[:, 0].max() + 1.0 # "
# Parameters for below...
steps = 1024
empty_lig_bias = 4.0
log_prob_cap = 32.0
# Array for collating dots so we can visualise the below...
dots = []
# For each line that has information...
for part in self.parts:
text = part['entry'].get_text().strip()
if len(text) == 0: continue
line_no = part['line'][0]
print 'Processing line %i...' % line_no
# Run dynamic programming...
## Work out what the states are, create a mapping from classes...
words = text.split()
punc_words = []
for word in words:
while len(word) != 0 and (word[0] not in letters):
punc_words.append(word[0])
word = word[1:]
tail = []
while len(word) != 0 and (word[-1] not in letters):
tail.append(word[-1])
word = word[:-1]
if len(word) != 0:
punc_words.append(word)
punc_words += tail
coded = '_'.join(map(lambda w: '_'.join(w), punc_words))
coded = '_' + coded + '_'
classed = numpy.array(
map(
lambda c: self.let.auto_model['classes'].index(c)
if c in self.let.auto_model['classes'] else 0, coded))
print coded
def word_to_wes(word):
if len(word) == 1:
if word not in letters: return 'b'
else: return 's' # s for snake - head and tail.
if len(word) == 2: return 'h t'
return 'h ' + ' '.join('b' * (
len(word) - 2)) + ' t' # h = head, b = body, t = tail.
wes = ' ' + ' '.join(map(word_to_wes, punc_words)) + ' '
print 'Found %i sections' % len(classed)
## Create the discrete dynamic programming solver...
dp = DDP()
dp.prepare(steps, len(classed))
## Do the unary terms...
print 'Calculating DP unary terms...'
uc = numpy.zeros((steps, classed.shape[0]),
dtype=numpy.float32)
uw = numpy.zeros(steps, dtype=numpy.float32)
code = start_cpp() + """
// Iterate each feature and if its on the line factor it into the unary cost...
for (int i=0; i<Nprobs[0]; i++)
{
float y = POS2(i, 1);
if ((y>=(line_no-0.5)) && (y<=(line_no+1.5)))
{
// Calculate the weight...
float weight = 1.0;
if (y<line_no) weight = 2.0 * (y - (line_no-0.5));
if (y>(line_no+1)) weight = 2.0 * ((line_no+1.5) - y);
// Identify its bin...
int bin = Nuc[0] * (POS2(i, 0) - float(min_x)) / (float(max_x) - float(min_x));
if (bin>=Nuc[0]) bin = Nuc[0] - 1;
// Sum it in, applying the mapping from sequence positions to classes (doing an incrimental mean)...
UW1(bin) += weight;
for (int j=0; j<Nuc[1]; j++)
{
float p = PROBS2(i, CLASSED1(j));
float nlp = -log(p);
if (nlp>float(log_prob_cap)) nlp = float(log_prob_cap);
UC2(bin, j) += (nlp - UC2(bin, j)) * weight / UW1(bin);
}
}
}
// Check all unary costs - any that are zero cost bias strongly towards being ligaments (Which are always class 0)...
for (int i=0; i<Nuc[0]; i++)
{
if (UW1(i)<1.0)
{
float bias = float(empty_lig_bias) * (1.0 - UW1(i));
// Nothing - bias to be a ligament...
for (int j=0; j<Nuc[1]; j++)
{
if (CLASSED1(j)!=0)
{
UC2(i, j) += bias;
}
}
}
}
"""
weave.inline(code, [
'line_no', 'min_x', 'max_x', 'empty_lig_bias',
'log_prob_cap', 'uc', 'uw', 'classed', 'probs', 'pos'
])
uc[0, 1:] = float('inf')
dp.unary(0, uc)
## Do the pairwise terms - weighted to encourage splits in such areas...
print 'Setting DP pairwise terms...'
tranc = numpy.clip(uw, 0.5, 1.0)
tranc = 2.0 * 0.5 * (tranc[:-1] + tranc[1:])
tranc = numpy.concatenate((numpy.zeros(
(steps - 1), dtype=numpy.float32)[numpy.newaxis, :],
tranc[numpy.newaxis, :])).T
dp.pairwise(0, ['ordered'] * (steps - 1), tranc)
## Solve...
print 'Dynamic programming...'
start = time.time()
best, cost = dp.best(classed.shape[0] - 1)
end = time.time()
print '...done in %.1f seconds' % (end - start)
print 'MAP cost = %.3f' % cost
# Convert the transitions to x positions...
splits = [min_x] # x coordinate of each split, plus bounds
for i in xrange(1, steps):
if best[i - 1] != best[i]:
val = (i / float(steps)) * (max_x - min_x) + min_x
splits.append(val)
splits.append(max_x)
# Print out information about each characters limits...
#for i in xrange(len(coded)):
# print 'char: %s :: (%.3f - %.3f)' % (coded[i], splits[i], splits[i+1])
# Find the centres of the letters and tag them...
final_tag = [None] * len(coded)
for i in xrange(len(coded)):
if coded[i] != '_':
# The tag we are applying...
if wes[i] == 'b': tag = coded[i]
elif wes[i] == 'h': tag = '_' + coded[i]
elif wes[i] == 't': tag = coded[i] + '_'
elif wes[i] == 's': tag = '_' + coded[i] + '_'
else:
raise RuntimeError(
'coded and wes vectors do not match - please slap Tom'
)
final_tag[i] = tag
# Measure the height of a line in pixels, so we can scale costs in the below...
top = hg.dot(
numpy.array([
0.5 * (splits[i] + splits[i + 1]), line_no, 1.0
]))
bot = hg.dot(
numpy.array([
0.5 * (splits[i] + splits[i + 1]),
line_no + 1.0, 1.0
]))
top /= top[2]
bot /= bot[2]
height = numpy.sqrt(numpy.square(top - bot).sum())
# Loop a bunch of positions on the vertical - select the one that is closest to the vertical - saves implimenting a closest to capsule method with a reasonable approximation...
best_n = None
for offset in numpy.linspace(0.0, 1.0, 11):
# Its position - center of bounding box...
tag_pos = numpy.array([
0.5 * (splits[i] + splits[i + 1]),
line_no + offset, 1.0
])
# Translate from line to space to pixel space...
tag_pos = hg.dot(tag_pos)
tag_pos /= tag_pos[2]
# Find the nearest point...
n = self.let.line.nearest(tag_pos[0], tag_pos[1])
# Adjust the cost with a center bias...
n = list(n)
n[0] += 0.25 * height * numpy.fabs(offset - 0.75)
# Decide if we are going to use it...
if best_n == None or best_n[0] > n[0]:
best_n = n
# Create...
self.let.line.add_tag(best_n[1], best_n[2], tag)
# Find the intersects of the split points and add the splits...
for si, split in enumerate(splits[2:-2]):
si += 2
# Generate end points of a line...
start = numpy.array([split, line_no - 0.5, 1.0])
end = numpy.array([split, line_no + 1.5, 1.0])
# Apply the homography to get back to image space...
start = hg.dot(start)
start /= start[2]
end = hg.dot(end)
end /= end[2]
# Do the intersect...
cuts = self.let.line.intersect(start[0], start[1], end[0],
end[1])
# Filter all cuts that do not seperate the relevant tags - this avoids cutting off tails, and generally only results in one possible cut being left...
e_before = final_tag[si - 1]
e_after = final_tag[si]
if e_before == None: e_before = final_tag[si - 2]
if e_after == None and (si + 1) < len(final_tag):
e_after = final_tag[si + 1]
def valid_cut(cut):
before, after = self.let.line.between(cut[0], cut[1])
# Note: Orientation is not necesarily the same order, so before and after could be swapped...
for b, a in [(before, after), (after, before)]:
if b == None:
b_ok = e_before == None
else:
b_ok = e_before == b[1]
if a == None:
a_ok = e_after == None
else:
a_ok = e_after == a[1]
if b_ok and a_ok: return True
return False
cuts = filter(valid_cut, cuts)
# Apply all remaining cuts (Generally only 1, but could be two for a double connection, e.g. double t)...
for cut in cuts:
self.let.line.add_split(cut[0], cut[1])
# Add this lines dynamic programming unary terms into the dots visualisation, so we can see whats driving the output...
## Calculate the matrix to go from a column-label coordinate system to the viewers coordinate system, plus an appropriate scale...
pre_scale = numpy.eye(3, dtype=numpy.float32)
pre_scale[0, 0] = (max_x - min_x) / steps
pre_scale[0, 2] = min_x
pre_scale[1, 1] = pre_scale[0, 0]
pre_scale[1, 2] = line_no + 1.5
to_dot = hg.dot(pre_scale)
size = 0.5 * to_dot[0, 0] / to_dot[2, 2]
## Create a dot for each cost value...
d = numpy.empty(((uc.shape[0] - 1) * uc.shape[1], 6),
dtype=numpy.float32)
max_cost = uc[1:, :].max()
d[:, 0], d[:, 1] = map(
lambda a: a.flatten(),
numpy.meshgrid(numpy.arange(1, uc.shape[0]),
numpy.arange(uc.shape[1]),
indexing='ij'))
d[:, 2] = size
d[:, 3] = numpy.clip(
(uc[1:, :].flatten() / max_cost) * 3.0 - 2.0, 0.0, 1.0)
d[:, 4] = numpy.clip(
(uc[1:, :].flatten() / max_cost) * 3.0 - 1.0, 0.0, 1.0)
d[:, 5] = numpy.clip(
(uc[1:, :].flatten() / max_cost) * 3.0 - 0.0, 0.0, 1.0)
for i in xrange(d.shape[0]):
p = to_dot.dot([d[i, 0], d[i, 1], 1.0])
p /= p[2]
d[i, 1] = p[0]
d[i, 0] = p[1]
dots.append(d)
# Recalculate the segments - above almost certainly messed them up...
print 'Recalculating segments...'
self.let.line.segment()
# Hack the i dot problem - find all untagged segments and link them to a suitable segment if there is one that makes sense - check for alternate segments with tags in the same vertical...
print 'Planning tittle assignment...'
to_create = [] # Two stage for computational sanity.
planned = dict() # To avoid double links.
vert_seg = self.let.line.get_segs()
pos = self.let.line.pos()
for seg in xrange(self.let.line.segments):
adj = self.let.line.adjacent(seg)
tags = self.let.line.get_tags(seg)
if len(adj) == 0 and len(tags) == 0:
# We have an island with no tags and no neighbours - see if we can find it a friend...
## Find all edges in the same vertical, with a little extra wiggle room added in - we convert to line space for this as it makes things sane...
bounds = self.let.line.get_bounds(
seg) # bounds = (min_x, max_x, min_y, max_y)
pnts = numpy.empty((4, 3), dtype=numpy.float32)
pnts[0, :] = ihg.dot([bounds[0], bounds[2], 1.0])
pnts[1, :] = ihg.dot([bounds[1], bounds[2], 1.0])
pnts[2, :] = ihg.dot([bounds[0], bounds[3], 1.0])
pnts[3, :] = ihg.dot([bounds[1], bounds[3], 1.0])
pnts /= pnts[:, 2:3]
search = [
pnts[:, 0].min(), pnts[:, 0].max(), pnts[:, 1].min(),
pnts[:, 1].max()
]
search[0] -= 0.25
search[1] += 0.25
search[2] -= 0.25
search[3] += 1.0
pnts[0, :] = hg.dot([search[0], search[2], 1.0])
pnts[1, :] = hg.dot([search[1], search[2], 1.0])
pnts[2, :] = hg.dot([search[0], search[3], 1.0])
pnts[3, :] = hg.dot([search[1], search[3], 1.0])
pnts /= pnts[:, 2:3]
search = [
pnts[:, 0].min(), pnts[:, 0].max(), pnts[:, 1].min(),
pnts[:, 1].max()
]
edges = self.let.line.within(*search)
## Convert edges into segment numbers, using a set to remove duplicates...
friends = set()
for es in edges:
for ei in xrange(
*es.indices(self.let.line.edge_count)):
edge = self.let.line.get_edge(ei)
friends.add(vert_seg[edge[0]])
friends.add(vert_seg[edge[1]])
friends.discard(seg)
if len(friends) == 0:
continue # Nothing to befriend - give up.
## Find out which segment is closest - use bounding boxes for this, and include a bias to prefer vertical closeness...
vertical_scale = 0.25
friend = None
friend_dist_sqr = None
for elem in friends:
elem_bounds = self.let.line.get_bounds(elem)
dx = (
bounds[0] - elem_bounds[1]
) if elem_bounds[1] < bounds[0] else (
(elem_bounds[0] -
bounds[1]) if bounds[1] < elem_bounds[0] else 0.0)
dy = (
bounds[2] - elem_bounds[3]
) if elem_bounds[3] < bounds[2] else (
(elem_bounds[2] -
bounds[3]) if bounds[3] < elem_bounds[2] else 0.0)
dy *= vertical_scale
dist_sqr = dx * dx + dy * dy
if friend == None or friend_dist_sqr > dist_sqr:
friend = elem
friend_dist_sqr = dist_sqr
## Check we have not already ordered this link...
key = (min(seg, friend), max(seg, friend))
if key in planned: continue
planned[key] = True
## Find the vertex closest to the others bounding box in each segment...
friend_bounds = self.let.line.get_bounds(friend)
rv = numpy.where(vert_seg == seg)[0]
dx = numpy.zeros(rv.shape, dtype=numpy.float32)
dy = numpy.zeros(rv.shape, dtype=numpy.float32)
tfw = pos[rv][:, 0] < friend_bounds[0]
dx[tfw] = friend_bounds[0] - pos[rv][tfw, 0]
tfw = pos[rv][:, 0] > friend_bounds[1]
dx[tfw] = pos[rv][tfw, 0] - friend_bounds[1]
tfw = pos[rv][:, 1] < friend_bounds[2]
dy[tfw] = friend_bounds[2] - pos[rv][tfw, 1]
tfw = pos[rv][:, 1] > friend_bounds[3]
dy[tfw] = pos[rv][tfw, 1] - friend_bounds[3]
v_seg = rv[numpy.argmin(
numpy.square(dx) + numpy.square(dy))]
rv = numpy.where(vert_seg == friend)[0]
if rv.shape[
0] == 0: # Posible, though indicative of a previous issue.
print 'Considered auto-link to a 0 vertex segment - weird'
continue
dx = numpy.zeros(rv.shape, dtype=numpy.float32)
dy = numpy.zeros(rv.shape, dtype=numpy.float32)
tfw = pos[rv][:, 0] < bounds[0]
dx[tfw] = bounds[0] - pos[rv][tfw, 0]
tfw = pos[rv][:, 0] > bounds[1]
dx[tfw] = pos[rv][tfw, 0] - bounds[1]
tfw = pos[rv][:, 1] < bounds[2]
dy[tfw] = bounds[2] - pos[rv][tfw, 1]
tfw = pos[rv][:, 1] > bounds[3]
dy[tfw] = pos[rv][tfw, 1] - bounds[3]
v_friend = rv[numpy.argmin(
numpy.square(dx) + numpy.square(dy))]
## Link these two vertices (which have to be turned into edges) together...
edges_seg = self.let.line.vertex_to_edges(v_seg)
edges_friend = self.let.line.vertex_to_edges(v_friend)
if len(edges_seg) != 0 and len(edges_friend) != 0:
t_seg = 0.0 if edges_seg[0][1] == False else 1.0
t_friend = 0.0 if edges_friend[0][1] == False else 1.0
to_create.append((edges_seg[0][0], t_seg,
edges_friend[0][0], t_friend))
print 'Going to linked segment %i to segment %i' % (
seg, friend)
# Apply the above and rebuild the segments...
print 'Enacting tittle assignment plan...'
for cmd in to_create:
self.let.line.add_link(*cmd)
self.let.line.segment()
print 'Auto-tagging done'
end_time = time.clock()
self.let.alg_time += end_time - start_time
def test_compile(self):
code = start_cpp(dual_hdp_ds) + """
State state;
"""
weave.inline(code, support_code=dual_hdp_ds)
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
import unittest
from params import Params
from solve_shared import State
from model import DocModel
from utils.start_cpp import start_cpp
from ds_link_cpp import ds_link_code
from scipy import weave
# Shared code used to Gibbs sample the model - provides operations used repeatedly by the sampling code. Note that this contains all the heavy code used by the system - the rest is basically just loops. Additionally the data structure code is prepended to this, so this is the only shared code...
shared_code = ds_link_code + start_cpp() + """
// Code for resampling a documents cluster assignment...
void ResampleDocumentCluster(State & state, Document & doc)
{
// If the document does not currently have a cluster then create one for it - let 'em cluster in non-initialisation iterations...
if (doc.GetCluster()==0)
{
ItemRef<Cluster,Conc> * newC = state.clusters.Append();
newC->Body().alpha = state.rho.alpha;
newC->Body().beta = state.rho.beta;
newC->Body().conc = state.rho.conc;
doc.SetCluster(newC);
return;
}
// Fill probAux of the topics with the counts of how many of each topic exist in the document whilst at the same time detaching the cluster instances from the document instances...
def __buildPyramid(self, base, pyramid):
"""Given an image and a pyramid as a list of images, largest first and of the same size as image, this fills in the images with a Gaussian pyramid."""
codeBlur = start_cpp() + """
// Calculate the filter - we just use 3 points as its a very tiny blur...
float filter[3];
filter[0] = exp(-0.5/(strength*strength));
filter[1] = 1.0;
filter[2] = filter[0];
float div = filter[0] + filter[1] + filter[2];
for (int f=0;f<3;f++) filter[f] /= div;
// Apply in the vertical dimension, going from img to temp...
for (int x=0;x<Nimg[1];x++)
{
for (int c=0;c<3;c++)
{
TEMP3(0,x,c) = (filter[0]+filter[1])*IMG3(0,x,c) + filter[2]*IMG3(1,x,c);
}
}
int ym1 = Nimg[0] - 1;
for (int y=1;y<ym1;y++)
{
for (int x=0;x<Nimg[1];x++)
{
for (int c=0;c<3;c++)
{
TEMP3(y,x,c) = filter[0]*IMG3(y-1,x,c) + filter[1]*IMG3(y,x,c) + filter[2]*IMG3(y+1,x,c);
}
}
}
for (int x=0;x<Nimg[1];x++)
{
for (int c=0;c<3;c++)
{
TEMP3(ym1,x,c) = filter[0]*IMG3(ym1-1,x,c) + (filter[1]+filter[2])*IMG3(ym1,x,c);
}
}
// Apply in the horizontal dimension, going from temp to img...
for (int y=0;y<Nimg[0];y++)
{
for (int c=0;c<3;c++)
{
IMG3(y,0,c) = (filter[0]+filter[1])*TEMP3(y,0,c) + filter[2]*TEMP3(y,1,c);
}
}
int xm1 = Nimg[1] - 1;
for (int y=0;y<Nimg[0];y++)
{
for (int x=1;x<xm1;x++)
{
for (int c=0;c<3;c++)
{
IMG3(y,x,c) = filter[0]*TEMP3(y,x-1,c) + filter[1]*TEMP3(y,x,c) + filter[2]*TEMP3(y,x+1,c);
}
}
}
for (int y=0;y<Nimg[0];y++)
{
for (int c=0;c<3;c++)
{
IMG3(y,xm1,c) = filter[0]*TEMP3(y,xm1-1,c) + (filter[1]+filter[2])*TEMP3(y,xm1,c);
}
}
"""
codeHalf = start_cpp() + """
// Iterate the output image and calculate each pixel in turn...
for (int y=0;y<NbOut[0];y++)
{
int sy = y*2;
bool safeY = sy+1<NbOut[0];
for (int x=0;x<NbOut[1];x++)
{
int sx = x*2;
bool safeX = sx+1<NbOut[1];
float div = 1.0;
for (int c=0;c<3;c++)
{
BOUT3(y,x,c) = BIN3(sy,sx,c);
}
if (safeX)
{
div += 1.0;
for (int c=0;c<3;c++) BOUT3(y,x,c) += BIN3(sy,sx+1,c);
}
if (safeY)
{
div += 1.0;
for (int c=0;c<3;c++) BOUT3(y,x,c) += BIN3(sy+1,sx,c);
}
if (safeX&&safeY)
{
div += 1.0;
for (int c=0;c<3;c++) BOUT3(y,x,c) += BIN3(sy+1,sx+1,c);
}
for (int c=0;c<3;c++) BOUT3(y,x,c) /= div;
}
}
"""
temp = self.image
strength = self.pyramidSD
pyramid[0][:, :, :] = base
img = pyramid[0][:, :, :]
weave.inline(codeBlur, ['img', 'temp', 'strength'])
for l in xrange(1, len(pyramid)):
bIn = pyramid[l - 1]
bOut = pyramid[l]
img = bOut
weave.inline(codeHalf, ['bIn', 'bOut'])
weave.inline(codeBlur, ['img', 'temp', 'strength'])
# Copyright (c) 2011, Tom SF Haines
# All rights reserved.
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions are met:
# * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL <COPYRIGHT HOLDER> BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
from scipy import weave
import unittest
from utils.start_cpp import start_cpp
# Defines code for a doubly linked list - simple but works as expected... (Includes its data via templated inheritance - a little strange, but neat and saves on memory thrashing.)
linked_list_code = start_cpp() + """
// Predefinitions...
template <typename ITEM, typename BODY> class Item;
template <typename ITEM, typename BODY> class List;
// Useful default...
struct Empty {};
// Item for the linked list data structure - simply inherits extra data stuff...
template <typename ITEM = Empty, typename BODY = Empty>
class Item : public ITEM
{
public:
Item(List<ITEM,BODY> * head):head(head),next(this),prev(this) {}
def test_compile(self):
code = start_cpp(shared_code) + """
"""
weave.inline(code, support_code=shared_code)
import unittest
from params import Params
from solve_shared import State
from model import DocModel
from utils.start_cpp import start_cpp
from ds_link_cpp import ds_link_code
from scipy import weave
# Shared code used to Gibbs sample the model - provides operations used repeatedly by the sampling code. Note that this contains all the heavy code used by the system - the rest is basically just loops. Additionally the data structure code is prepended to this, so this is the only shared code...
shared_code = ds_link_code + start_cpp() + """
// Code for resampling a documents cluster assignment...
void ResampleDocumentCluster(State & state, Document & doc)
{
// If the document does not currently have a cluster then create one for it - let 'em cluster in non-initialisation iterations...
if (doc.GetCluster()==0)
{
ItemRef<Cluster,Conc> * newC = state.clusters.Append();
newC->Body().alpha = state.rho.alpha;
newC->Body().beta = state.rho.beta;
newC->Body().conc = state.rho.conc;
doc.SetCluster(newC);
return;
}
// Fill probAux of the topics with the counts of how many of each topic exist in the document whilst at the same time detaching the cluster instances from the document instances...
def test_compile(self): code = start_cpp(shared_code) + """ """ weave.inline(code, support_code=shared_code)
def errorC(goal, gen, es, esclName = 'es'):
"""Provides C code that can be used by the error method to go much faster. Makes use of a goal, a generator, and an exampler set, and the code will be unique for each keying combination of these. Will return None if C code generation is not supported for the particular combination."""
# First do accessors for the data set...
try:
escl = es.listCodeC(esclName)
except NotImplementedError: return None
code = ''
for channel in escl:
code += channel['get'] + '\n'
code += channel['exemplars'] + '\n'
code += channel['features'] + '\n'
# Throw in the test code...
try:
code += gen.testCodeC('do_test', escl) + '\n'
except NotImplementedError: return None
# Definition of Exemplar...
code += start_cpp() + """
// So we can use an inplace modified linkied list to avoid malloc's during the real work...
struct Exemplar
{
int index;
float weight;
Exemplar * next;
};
"""
# Add the needed goal code...
try:
gDic = goal.codeC('goal', escl)
except NotImplementedError:
return None
try:
code += gDic['summary']
code += gDic['updateSummary']
code += gDic['error']
except KeyError:
return None
# The actual code...
code += start_cpp() + """
// Recursivly calculates the error whilst updating the summaries...
// node - node of the tree; for an external user this will always be the root.
// data - python tuple containing the inputs needed at each stage.
// test_set - Linked list of entities to use to generate/update the error.
// err - Variable into which the error will be output. Must be 0.0 on call.
// weight - Weight that can be used in the error calculation - basically temporary storage. Must be 0.0 on call.
void error(PyObject * node, PyObject * data, Exemplar * test_set, float & err, float & weight, bool inc)
{
// Calculate/update the summary at this node, but only store it if inc is true...
void * sum = 0;
size_t sumLen = 0;
PyObject * summary = PyObject_GetAttrString(node, "summary");
if (summary==Py_None)
{
goal_summary(data, test_set, sum, sumLen);
}
else
{
sumLen = PyString_Size(summary);
sum = realloc(sum, sumLen);
memcpy(sum, PyString_AsString(summary), sumLen);
goal_updateSummary(data, test_set, sum, sumLen);
}
Py_DECREF(summary);
if (inc)
{
PyObject * t = PyString_FromStringAndSize((char*)sum, sumLen);
PyObject_SetAttrString(node, "summary", t);
Py_DECREF(t);
}
// If there is a test then recurse, otherwise calculate and include the error...
PyObject * test = PyObject_GetAttrString(node, "test");
if (test==Py_None)
{
// Leaf node - calculate and store the error...
PyObject * stats = PyObject_GetAttrString(node, "stats");
void * s = PyString_AsString(stats);
size_t sLen = PyString_Size(stats);
goal_error(s, sLen, sum, sumLen, err, weight);
Py_DECREF(stats);
}
else
{
// Branch node - use the test to split the test_set and recurse...
// Tests...
Exemplar * pass = 0;
Exemplar * fail = 0;
void * test_ptr = PyString_AsString(test);
size_t test_len = PyString_Size(test);
while (test_set)
{
Exemplar * next = test_set->next;
if (do_test(data, test_ptr, test_len, test_set->index))
{
test_set->next = pass;
pass = test_set;
}
else
{
test_set->next = fail;
fail = test_set;
}
test_set = next;
}
// Recurse...
if ((pass!=0)||inc)
{
PyObject * child = PyObject_GetAttrString(node, "true");
error(child, data, pass, err, weight, inc);
Py_DECREF(child);
}
if ((fail!=0)||inc)
{
PyObject * child = PyObject_GetAttrString(node, "false");
error(child, data, fail, err, weight, inc);
Py_DECREF(child);
}
}
// Clean up...
Py_DECREF(test);
free(sum);
}
"""
return code
# Copyright 2011 Tom SF Haines
# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at
# http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.
from scipy import weave
import unittest
from utils.start_cpp import start_cpp
from dp_utils.dp_utils import dp_utils_code
# Data structure for storing the state of the model, for use with the c++ Gibbs sampling code. A custom structure is used for speed and to keep the code clean...
ds_code = dp_utils_code + start_cpp() + """
// Details specific for a topic - basically its multinomial and some helper stuff...
class Topic
{
public:
Topic():wc(0) {}
~Topic() {delete[] wc;}
int id; // Used when moving this data structure to and from python.
int * wc; // Indexed by word id this contains the count of how many words with that id are assigned to this topic - from the prior the multinomial can hence be worked out.
int wcTotal; // Sum of above.
float prob; // Helper for resampling.
float probAux; // "
};
