def add_condition(self,feature,condition,threshold):
from globfile import args,buckets
if args['d']:
self.features.append(feature)
self.conditions.append(condition)
buck_fea = [ i for i in buckets.keys() if feature == i[1:]][0]
if condition:
if int(threshold) in buckets[buck_fea].hi.keys():
self.thresholds.append(buckets[buck_fea].hi[int(threshold)])
else:
self.thresholds.append(buckets[buck_fea].hi[0])
else:
if int(threshold) in buckets[buck_fea].lo.keys():
self.thresholds.append(buckets[buck_fea].lo[int(threshold)])
else:
self.thresholds.append(buckets[buck_fea].lo[0])
else:
self.features.append(feature)
self.conditions.append(condition)
self.thresholds.append(threshold)
def add_condition(self, feature, condition, threshold):
from globfile import args, buckets
if args['d']:
self.features.append(feature)
self.conditions.append(condition)
buck_fea = [i for i in buckets.keys() if feature == i[1:]][0]
if condition:
if int(threshold) in buckets[buck_fea].hi.keys():
self.thresholds.append(
buckets[buck_fea].hi[int(threshold)])
else:
self.thresholds.append(buckets[buck_fea].hi[0])
else:
if int(threshold) in buckets[buck_fea].lo.keys():
self.thresholds.append(
buckets[buck_fea].lo[int(threshold)])
else:
self.thresholds.append(buckets[buck_fea].lo[0])
else:
self.features.append(feature)
self.conditions.append(condition)
self.thresholds.append(threshold)
def tshortener(z,zlst,colname,data,dep,indep,patt=1.0,discretize=True):
#The infogain techniques of pruning columns and discretization
class Bucket:
#class for each column with splitted pairs of data
def __init__(self,name):
self.pairs = [] #unsorted row pairs
self.name = name
self.wsum = 0
self.dinds = {} #sorted split indexs
self.lo = {}
self.hi = {}
def addpairs(self,pairs):
self.pairs.append(pairs)
def addwsum(self,wsum):
self.wsum = wsum
def __repr__(self):
s = 'n: '+str(self.name)+":"
s += ' l: '+str(len(self.pairs))
s += ' e: '+str(self.wsum)+'\n'
return s
from globfile import buckets
outcols = []
for key,value in buckets.items():
buckets[key] = None
for Z in zlst[1:]:
for c in indep[Z]:
if c == 'C_id': continue
if c not in buckets.keys():
buckets[c] = Bucket(c)
elif buckets[c] is None:
buckets[c] = Bucket(c)
ind = colname[Z].index(c)
cind = colname[Z].index('C_id')
for r in data[Z]:
buckets[c].addpairs((r[ind],str(r[cind])))
reader.removeTable(Z)
buckets = weighted_entropies(buckets)
vals = buckets.values()[:]
vals.sort(key=lambda x: x.wsum,reverse=False)
for i in range(0,int(len(vals)*patt)):
outcols.append(vals[i].name)
zshort = 'shortenedz'
outcols = [i for i in colname[z] if i in outcols]
print outcols,"#infogained"
#Convert outcols to discrete attributes
if discretize:
outcols = [c[1:] for c in outcols]
print outcols,"#discretized"
reader.makeTable(outcols+dep[z],zshort)
for r in data[z]:
temp = []
for i,c in enumerate(colname[z]):
if discretize:
if c[1:] in outcols or c in dep[z]:
temp.append(r[i])
else:
if c in outcols+dep[z]:
temp.append(r[i])
reader.addRow(temp,zshort)
if discretize: discretizer(zshort,buckets)
for Z in zlst:
reader.removeTable(Z)
#discretizer(zshort,buckets)
return zshort
def tshortener(z, zlst, colname, data, dep, indep, patt=1.0, discretize=True):
#The infogain techniques of pruning columns and discretization
class Bucket:
#class for each column with splitted pairs of data
def __init__(self, name):
self.pairs = [] #unsorted row pairs
self.name = name
self.wsum = 0
self.dinds = {} #sorted split indexs
self.lo = {}
self.hi = {}
def addpairs(self, pairs):
self.pairs.append(pairs)
def addwsum(self, wsum):
self.wsum = wsum
def __repr__(self):
s = 'n: ' + str(self.name) + ":"
s += ' l: ' + str(len(self.pairs))
s += ' e: ' + str(self.wsum) + '\n'
return s
from globfile import buckets
outcols = []
for key, value in buckets.items():
buckets[key] = None
for Z in zlst[1:]:
for c in indep[Z]:
if c == 'C_id': continue
if c not in buckets.keys():
buckets[c] = Bucket(c)
elif buckets[c] is None:
buckets[c] = Bucket(c)
ind = colname[Z].index(c)
cind = colname[Z].index('C_id')
for r in data[Z]:
buckets[c].addpairs((r[ind], str(r[cind])))
reader.removeTable(Z)
buckets = weighted_entropies(buckets)
vals = buckets.values()[:]
vals.sort(key=lambda x: x.wsum, reverse=False)
for i in range(0, int(len(vals) * patt)):
outcols.append(vals[i].name)
zshort = 'shortenedz'
outcols = [i for i in colname[z] if i in outcols]
print outcols, "#infogained"
#Convert outcols to discrete attributes
if discretize:
outcols = [c[1:] for c in outcols]
print outcols, "#discretized"
reader.makeTable(outcols + dep[z], zshort)
for r in data[z]:
temp = []
for i, c in enumerate(colname[z]):
if discretize:
if c[1:] in outcols or c in dep[z]:
temp.append(r[i])
else:
if c in outcols + dep[z]:
temp.append(r[i])
reader.addRow(temp, zshort)
if discretize: discretizer(zshort, buckets)
for Z in zlst:
reader.removeTable(Z)
#discretizer(zshort,buckets)
return zshort
