def distance_pruner(zlst): #Prunes cluster tree i.e. zlst with distance between their centroids if args['v'] > -1: sys.stderr.write("\n#Pruning data based on eucledian distance between clusters.\n") print zlst," #Old zlst before distprune" import dist z0 = zlst[0] pairs = [] for _i,i in enumerate(data[z0]): for _j,j in enumerate(data[z0]): if i != j: if dist.dist(i,j,z0,indep,nump) < 0.3: if [_i,_j] not in pairs and [_j,_i] not in pairs: pairs.append(['__'+str(_i+1),'__'+str(_j+1)]) def repaired(pairs): for i in pairs: for j in pairs: if i != j : if i[0] in j or i[1] in j: pairs = [list(set(i+j))]+\ [k for k in pairs if k not in [i,j]] return repaired(pairs) return pairs repairs = repaired(pairs) ps = [] for i in repairs: ps+=i for i in zlst[1:]: if i not in ps: repairs.append([i]) temp_row = {} for ind,p in enumerate(repairs): temp_row[ind] = [] for i in p: temp_row[ind] += data[i] col = colname[z0] for Z in zlst: reader.removeTable(Z) zlst = [None] for ind,value in enumerate(temp_row.values()): Z = '__'+str(ind+1) reader.makeTable(col,Z) for r in value: reader.addRow(r[:len(r)-1]+[ind],Z) zlst.append(Z) xy_lib.buildzero(zlst,'',args['e']) if args['v'] > -1: print zlst," #New zlst after distprune" return zlst
def distance_pruner(zlst): #Prunes cluster tree i.e. zlst with distance between their centroids if args['v'] > -1: sys.stderr.write( "\n#Pruning data based on eucledian distance between clusters.\n") print zlst, " #Old zlst before distprune" import dist z0 = zlst[0] pairs = [] for _i, i in enumerate(data[z0]): for _j, j in enumerate(data[z0]): if i != j: if dist.dist(i, j, z0, indep, nump) < 0.3: if [_i, _j] not in pairs and [_j, _i] not in pairs: pairs.append(['__' + str(_i + 1), '__' + str(_j + 1)]) def repaired(pairs): for i in pairs: for j in pairs: if i != j: if i[0] in j or i[1] in j: pairs = [list(set(i+j))]+\ [k for k in pairs if k not in [i,j]] return repaired(pairs) return pairs repairs = repaired(pairs) ps = [] for i in repairs: ps += i for i in zlst[1:]: if i not in ps: repairs.append([i]) temp_row = {} for ind, p in enumerate(repairs): temp_row[ind] = [] for i in p: temp_row[ind] += data[i] col = colname[z0] for Z in zlst: reader.removeTable(Z) zlst = [None] for ind, value in enumerate(temp_row.values()): Z = '__' + str(ind + 1) reader.makeTable(col, Z) for r in value: reader.addRow(r[:len(r) - 1] + [ind], Z) zlst.append(Z) xy_lib.buildzero(zlst, '', args['e']) if args['v'] > -1: print zlst, " #New zlst after distprune" return zlst
def tshortener(z,zlst,colname,data,dep,indep,patt=1.0,discretize=True): #The infogain techniques of pruning columns and discretization class Bucket: #class for each column with splitted pairs of data def __init__(self,name): self.pairs = [] #unsorted row pairs self.name = name self.wsum = 0 self.dinds = {} #sorted split indexs self.lo = {} self.hi = {} def addpairs(self,pairs): self.pairs.append(pairs) def addwsum(self,wsum): self.wsum = wsum def __repr__(self): s = 'n: '+str(self.name)+":" s += ' l: '+str(len(self.pairs)) s += ' e: '+str(self.wsum)+'\n' return s from globfile import buckets outcols = [] for key,value in buckets.items(): buckets[key] = None for Z in zlst[1:]: for c in indep[Z]: if c == 'C_id': continue if c not in buckets.keys(): buckets[c] = Bucket(c) elif buckets[c] is None: buckets[c] = Bucket(c) ind = colname[Z].index(c) cind = colname[Z].index('C_id') for r in data[Z]: buckets[c].addpairs((r[ind],str(r[cind]))) reader.removeTable(Z) buckets = weighted_entropies(buckets) vals = buckets.values()[:] vals.sort(key=lambda x: x.wsum,reverse=False) for i in range(0,int(len(vals)*patt)): outcols.append(vals[i].name) zshort = 'shortenedz' outcols = [i for i in colname[z] if i in outcols] print outcols,"#infogained" #Convert outcols to discrete attributes if discretize: outcols = [c[1:] for c in outcols] print outcols,"#discretized" reader.makeTable(outcols+dep[z],zshort) for r in data[z]: temp = [] for i,c in enumerate(colname[z]): if discretize: if c[1:] in outcols or c in dep[z]: temp.append(r[i]) else: if c in outcols+dep[z]: temp.append(r[i]) reader.addRow(temp,zshort) if discretize: discretizer(zshort,buckets) for Z in zlst: reader.removeTable(Z) #discretizer(zshort,buckets) return zshort
def tshortener(z, zlst, colname, data, dep, indep, patt=1.0, discretize=True): #The infogain techniques of pruning columns and discretization class Bucket: #class for each column with splitted pairs of data def __init__(self, name): self.pairs = [] #unsorted row pairs self.name = name self.wsum = 0 self.dinds = {} #sorted split indexs self.lo = {} self.hi = {} def addpairs(self, pairs): self.pairs.append(pairs) def addwsum(self, wsum): self.wsum = wsum def __repr__(self): s = 'n: ' + str(self.name) + ":" s += ' l: ' + str(len(self.pairs)) s += ' e: ' + str(self.wsum) + '\n' return s from globfile import buckets outcols = [] for key, value in buckets.items(): buckets[key] = None for Z in zlst[1:]: for c in indep[Z]: if c == 'C_id': continue if c not in buckets.keys(): buckets[c] = Bucket(c) elif buckets[c] is None: buckets[c] = Bucket(c) ind = colname[Z].index(c) cind = colname[Z].index('C_id') for r in data[Z]: buckets[c].addpairs((r[ind], str(r[cind]))) reader.removeTable(Z) buckets = weighted_entropies(buckets) vals = buckets.values()[:] vals.sort(key=lambda x: x.wsum, reverse=False) for i in range(0, int(len(vals) * patt)): outcols.append(vals[i].name) zshort = 'shortenedz' outcols = [i for i in colname[z] if i in outcols] print outcols, "#infogained" #Convert outcols to discrete attributes if discretize: outcols = [c[1:] for c in outcols] print outcols, "#discretized" reader.makeTable(outcols + dep[z], zshort) for r in data[z]: temp = [] for i, c in enumerate(colname[z]): if discretize: if c[1:] in outcols or c in dep[z]: temp.append(r[i]) else: if c in outcols + dep[z]: temp.append(r[i]) reader.addRow(temp, zshort) if discretize: discretizer(zshort, buckets) for Z in zlst: reader.removeTable(Z) #discretizer(zshort,buckets) return zshort