def add2set(gset, elist, H): n = len(H) s = set() ss = set() eremove = {e: True for e in elist} for gnum in gset: g = bfu.num2CG(gnum, n) for e in elist: if not e[1] in g[e[0]]: gk.addanedge(g,e) num = bfu.g2num(g) if not num in s: au = bfu.call_undersamples(g) if not gk.checkconflict(H, g, au=au): eremove[e] = False s.add(num) if gk.checkequality(H, g, au=au): ss.add(num) gk.delanedge(g,e) for e in eremove: if eremove[e]: elist.remove(e) return s, ss, elist
def simple_loops(g, u): """ iterator over the list of simple loops of graph g at the undersample rate u """ gx = graph2nx(num2CG(g2num(undersample(g, u)), len(g))) for l in networkx.simple_cycles(gx): yield l
def edgemask(gl, H, cds): nl= len(gl) ss = set() mask = np.zeros((nl,nl),'int') np.fill_diagonal(mask,-1) idx = np.triu_indices(nl,1) for i,j in zip(idx[0], idx[1]): if gl[i] & gl[j]: mask[i,j] = -1 mask[j,i] = -1 continue if skip_conflict(gl[i], gl[j], cds): gnum = gl[i] | gl[j] cache[gnum] = False continue gnum = gl[i] | gl[j] if gnum in cache: if cache[gnum]: mask[i,j] = gnum mask[j,i] = gnum else: cache[gnum] = False g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): ss.add(gnum) mask[i,j] = gnum mask[j,i] = gnum cache[gnum] = True return mask, ss
def simple_loops(g, u): """ iterator over the list of simple loops of graph g at the undersample rate u """ gx = graph2nx(num2CG(g2num(undersample(g,u)), len(g))) for l in networkx.simple_cycles(gx): yield l
def ledgemask(gl,H, cds): """given a list of encoded graphs and observed undersampled graph H returns a matrix with -1 on diagonal, 0 at the conflicting graph combination and encoded graph at non-conflicted positions. Furthermore, returns a set of graphs that are in the equivalence class of H Arguments: - `gl`: list of integer encoded graphs - `H`: the observed undersampled graph """ n = len(H) nl= len(gl) s = set() mask = np.zeros((nl,nl),'int') np.fill_diagonal(mask,-1) for i in xrange(nl): for j in xrange(i+1,nl): if gl[i] & gl[j]: continue gnum = gl[i] | gl[j] if skip_conflictors(gnum, cds): continue g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): s.add(gnum) mask[i,j] = gnum mask[j,i] = gnum return mask, s
def del_loop(ds, H, iter=0, verbose=True, capsize=1000): n = len(H) dsr = {} s = set() ss = set() print iter, for gnum in ds: gset = [] s = set() for sloop in ds[gnum]: rset = ds[gnum] - set([sloop]) num = reduce(operator.or_, rset) if not num in s: g = bfu.num2CG(num, n) if bfu.overshoot(g, H): s.add(num) gset.append((num,rset)) if gset == []: print '.', ss.add(gnum) for gn in gset: dsr[gn[0]] = gn[1] print '' return dsr, ss
def edgemask(gl, H, cds): nl = len(gl) ss = set() mask = np.zeros((nl, nl), 'int') np.fill_diagonal(mask, -1) idx = np.triu_indices(nl, 1) for i, j in zip(idx[0], idx[1]): if gl[i] & gl[j]: mask[i, j] = -1 mask[j, i] = -1 continue if skip_conflict(gl[i], gl[j], cds): gnum = gl[i] | gl[j] cache[gnum] = False continue gnum = gl[i] | gl[j] if gnum in cache: if cache[gnum]: mask[i, j] = gnum mask[j, i] = gnum else: cache[gnum] = False g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): ss.add(gnum) mask[i, j] = gnum mask[j, i] = gnum cache[gnum] = True return mask, ss
def delAloop(g, loop): n = len(g) l = [] l = [bfu.g2num(ur.loop2graph(s,n)) for s in sls.simple_loops(g,0)] l = [num for num in l if not num == loop ] print loop, ': ', l return bfu.num2CG(reduce(operator.or_, l),n)
def build_loop_step(ds, loop, n, iter=1): n2 = n*n +n dsr = {} s = set() ss = set() pbar = start_progress_bar(iter, len(ds)) c = 0 for gnum in ds: c += 1 pbar.update(c) gset = set() eset = set() for sloop in ds[gnum]: num = sloop | gnum if not num in s: g = bfu.num2CG(num, n) s.add(num) if bfu.forms_loop(g, loop): ss.add(num) else: gset.add((num,sloop)) eset.add(sloop) for gn,e in gset: dsr[gn] = eset - set([e]) pbar.finish() return dsr, ss
def add2set(gset, elist, H): n = len(H) s = set() ss = set() eremove = {e: True for e in elist} for gnum in gset: g = bfu.num2CG(gnum, n) for e in elist: if not e[1] in g[e[0]]: gk.addanedge(g, e) num = bfu.g2num(g) if not num in s: au = bfu.call_undersamples(g) if not gk.checkconflict(H, g, au=au): eremove[e] = False s.add(num) if gk.checkequality(H, g, au=au): ss.add(num) gk.delanedge(g, e) for e in eremove: if eremove[e]: elist.remove(e) return s, ss, elist
def edgemask(gl, H, cds): """given a list of encoded graphs and observed undersampled graph H returns a matrix with -1 on diagonal, 0 at the conflicting graph combination and encoded graph at non-conflicted positions. Furthermore, returns a set of graphs that are in the equivalence class of H Arguments: - `gl`: list of integer encoded graphs - `H`: the observed undersampled graph """ n = len(H) nl = len(gl) s = set() mask = np.zeros((nl, nl), 'int') np.fill_diagonal(mask, -1) for i in xrange(nl): for j in xrange(i + 1, nl): if gl[i] & gl[j]: continue if skip_conflict(gl[i], gl[j], cds): continue gnum = gl[i] | gl[j] g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): s.add(gnum) mask[i, j] = gnum mask[j, i] = gnum return mask, s
def prune_loops(loops, H): l = [] n = len(H) for loop in loops: g = bfu.num2CG(loop, n) x = [k for k in g if g[k]] if len(x) == 1: s = reduce(lambda x, s: s.union(x), [H[x[0]][w] for w in H[x[0]]]) if not (2,0) in s: continue if not bfu.call_u_conflicts_d(g, H): l.append(loop) return l
def lconfpairs(H, cap=10, sloops=None): n = len(H) d = {} if not sloops: sloops = prune_loops(allsloops(len(H)),H) c = 0 for p in combinations(sloops,2): g = bfu.num2CG(p[0]|p[1], n) if bfu.call_u_conflicts(g, H): d.setdefault(p[0],set()).add(p[1]) d.setdefault(p[1],set()).add(p[0]) if c >= cap: break c +=1 return d
def prune(gl, H): l = [] ss = set() for gnum in gl: if gnum in cache: if cache[gnum]: l.append(gnum) else: cache[gnum] = False g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): ss.add(gnum) l.append(gnum) cache[gnum] = True return l
def matchAloop(loop, n): """returns a set of minimal graphs that generate this loop Arguments: - `loop`: binary encoding of the loop - `n`: number of nodes in the graph """ s = set() l = forward_loop_match(loop,n) print len(l) for g in l: s = s | reverse_edge_match(bfu.num2CG(g,n),loop) return s
def prune(gl, H): l = [] ss = set() for gnum in gl: if gnum in cache: if cache[gnum]: l.append(gnum) else: cache[gnum] = False g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): ss.add(gnum) l.append(gnum) cache[gnum] = True return l
def add2set_(ds, H, cp, ccf, iter=1, verbose=True, capsize=100): n = len(H) n2 = n*n +n dsr = {} s = set() ss = set() pbar = start_progress_bar(iter, len(ds), verbose = verbose) c = 0 for gnum in ds: g = bfu.num2CG(gnum, n) c += 1 pbar.update(c) glist = [] elist = [] eset = set() for e in ds[gnum]: if not e[1] in g[e[0]]: gk.addanedge(g,e) num = bfu.g2num(g) ekey = (1<<(n2 - int(e[0],10)*n - int(e[1],10))) if ekey in ccf and skip_conflictors(num,ccf[ekey]): gk.delanedge(g,e) continue if not num in s: s.add(num) if not bfu.call_u_conflicts(g, H): #cf, gl2 = bfu.call_u_conflicts2(g, H) #if not cf: glist.append((num,ekey)) elist.append(e) eset.add(ekey) if bfu.call_u_equals(g, H): ss.add(num) #if bfu.call_u_equals2(g, gl2, H): ss.add(num) if capsize <= len(ss): break gk.delanedge(g,e) for gn,e in glist: if e in cp: dsr[gn] = [ekey2e(k,n) for k in eset - cp[e]] else: dsr[gn] = elist if capsize <= len(ss): return dsr, ss pbar.finish() return dsr, ss
def add2set_(ds, H, cp, ccf, iter=1, verbose=True, capsize=100): n = len(H) n2 = n * n + n dsr = {} s = set() ss = set() pbar = start_progress_bar(iter, len(ds), verbose=verbose) c = 0 for gnum in ds: g = bfu.num2CG(gnum, n) gnum = bfu.g2num(g) c += 1 pbar.update(c) glist = [] elist = [] eset = set() for e in ds[gnum]: if not e[1] in g[e[0]]: gk.addanedge(g, e) num = bfu.g2num(g) ekey = (1 << (n2 - int(e[0], 10) * n - int(e[1], 10))) if ekey in ccf and skip_conflictors(num, ccf[ekey]): gk.delanedge(g, e) continue if not num in s: if not bfu.call_u_conflicts(g, H): #cf, gl2 = bfu.call_u_conflicts2(g, H) #if not cf: glist.append((num, ekey)) elist.append(e) eset.add(ekey) s.add(num) if bfu.call_u_equals(g, H): ss.add(num) if capsize <= len(ss): break #if bfu.call_u_equals2(g, gl2, H): ss.add(num) gk.delanedge(g, e) for gn, e in glist: if e in cp: dsr[gn] = [ekey2e(k, n) for k in eset - cp[e]] else: dsr[gn] = elist if capsize <= len(ss): return dsr, ss pbar.finish() return dsr, ss
def patchmerge(ds, H, cds): n = len(H) l = set() s = set() o = set() for gkey in ds: for num in ds[gkey]: if gkey & num: continue gnum = gkey | num if gnum is s: continue if skip_conflictors(gnum, cds): continue g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): l.add(gnum) if bfu.call_u_equals(g, H): s.add(gnum) elif not gnum in o and bfu.overshoot(g, H): o.add(gnum) return l, s, o
def patchmerge(ds, H, cds): n = len(H) l = set() s = set() o = set() for gkey in ds: for num in ds[gkey]: if gkey & num: continue gnum = gkey | num if gnum is s: continue if skip_conflictors(gnum, cds): continue g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): l.add(gnum) if bfu.call_u_equals(g, H): s.add(gnum) elif not gnum in o and bfu.overshoot(g, H): o.add(gnum) return l, s, o
def dceqclass2(H): """Find all graphs in the same equivalence class with respect to H Arguments: - `H`: an undersampled graph """ if cmp.isSclique(H): print 'not running on superclique' return set() n = len(H) s = set() cp = confpairs(H) confs = conflictor_set(H) ccf = conflictors(H) def prune_loops(gl, H): l = [] for e in gl: if e[0] == e[1] and not (e[1] in H[e[0]] and (1, 0) in H[e[0]][e[1]]): continue l.append(e) return l edges = gk.edgelist(gk.complement(bfu.num2CG(0, n))) edges = prune_loops(edges, H) glist = map(lambda x: e2num(x, n), edges) #glist = list(2**np.arange(n**2)) i = 0 while glist != []: print 2**i, len(glist) glist_prev = glist glist, ss = quadmerge21(glist, H, confs) s = s | ss i += 1 ds = {x: edges for x in glist_prev} for j in range(i, len(H)**2): ds, ss = add2set_(ds, H, cp, ccf, iter=j, verbose=True) s = s | ss if not ds: break return s
def edgemask2(gl,H, cds): n = len(H) nl= len(gl) s = set() o = set() mask = np.zeros((nl,nl),'int') np.fill_diagonal(mask,-1) for i in xrange(nl): for j in xrange(i+1,nl): if gl[i] & gl[j]: continue gnum = gl[i] | gl[j] if skip_conflictors(gnum, cds): continue g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): s.add(gnum) mask[i,j] = gnum mask[j,i] = gnum elif bfu.overshoot(g, H): o.add(gnum) return mask, s, o # mask, found eqc members, overshoots
def dceqclass2(H): """Find all graphs in the same equivalence class with respect to H Arguments: - `H`: an undersampled graph """ if cmp.isSclique(H): print 'not running on superclique' return set() n = len(H) s = set() cp = confpairs(H) confs = conflictor_set(H) ccf = conflictors(H) def prune_loops(gl, H): l = [] for e in gl: if e[0] == e[1] and not (e[1] in H[e[0]] and (1,0) in H[e[0]][e[1]]): continue l.append(e) return l edges = gk.edgelist(gk.complement(bfu.num2CG(0,n))) edges = prune_loops(edges, H) glist = map(lambda x: e2num(x,n),edges) #glist = list(2**np.arange(n**2)) i = 0 while glist != []: print 2**i, len(glist) glist_prev = glist glist, ss = quadmerge21(glist, H, confs) s = s|ss i += 1 ds = {x: edges for x in glist_prev} for j in range(i, len(H)**2): ds, ss = add2set_(ds, H, cp, ccf, iter=j, verbose=True) s = s | ss if not ds: break return s
def edgemask2(gl, H, cds): n = len(H) nl = len(gl) s = set() o = set() mask = np.zeros((nl, nl), 'int') np.fill_diagonal(mask, -1) for i in xrange(nl): for j in xrange(i + 1, nl): if gl[i] & gl[j]: continue gnum = gl[i] | gl[j] if skip_conflictors(gnum, cds): continue g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): if bfu.call_u_equals(g, H): s.add(gnum) mask[i, j] = gnum mask[j, i] = gnum elif bfu.overshoot(g, H): o.add(gnum) return mask, s, o # mask, found eqc members, overshoots
def quadmerge_(glist, H, ds): n = len(H) gl = set() ss = set() conflicts = set() for gi in combinations(glist, 2): if gi[0] & gi[1]: continue #if skip_conflict(gi[0], gi[1], ds): continue gnum = gi[0] | gi[1] if gnum in conflicts: continue if skip_conflictors(gnum, ds): conflicts.add(gnum) continue if gnum in gl: continue g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): gl.add(gnum) if bfu.call_u_equals(g, H): ss.add(gnum) else: conflicts.add(gnum) return gl, ss
def quadmerge_(glist, H, ds): n = len(H) gl = set() ss = set() conflicts = set() for gi in combinations(glist, 2): if gi[0] & gi[1]: continue #if skip_conflict(gi[0], gi[1], ds): continue gnum = gi[0] | gi[1] if gnum in conflicts: continue if skip_conflictors(gnum, ds): conflicts.add(gnum) continue if gnum in gl: continue g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): gl.add(gnum) if bfu.call_u_equals(g, H): ss.add(gnum) else: conflicts.add(gnum) return gl, ss
def add2set_loop(ds, H, cp, ccf, iter=1, verbose=True, capsize=100, currsize=0): n = len(H) n2 = n*n +n dsr = {} s = set() ss = set() pbar = start_progress_bar(iter, len(ds), verbose = verbose) c = 0 for gnum in ds: c += 1 pbar.update(c) gset = set() eset = set() for sloop in ds[gnum]: if sloop & gnum == sloop: continue num = sloop | gnum if sloop in ccf and skip_conflictors(num,ccf[sloop]):continue if not num in s: g = bfu.num2CG(num, n) if not bfu.call_u_conflicts(g, H): s.add(num) gset.add((num,sloop)) eset.add(sloop) if bfu.call_u_equals(g, H): ss.add(num) if capsize <= len(ss)+currsize: return dsr, ss #for gn,e in gset: # if e in cp: # dsr[gn] = eset - cp[e] - set([e]) # else: # dsr[gn] = eset - set([e]) for gn in gset: dsr[gn[0]] = eset - set([gn[1]]) pbar.finish() return dsr, ss
def lquadmerge(gl, H, cds): n = len(H) l = set() s = set() mask, ss = ledgemask(gl, H, cds) s = s | ss ds = edgeds(mask) #pp = pprint.PrettyPrinter(indent=1) #pp.pprint(ds) for idx in ds: for gn in ds[idx]: if mask[idx]&gn: continue if skip_conflictors(mask[idx], gn, cds): continue gnum = mask[idx] | gn if gnum in l or gnum in ss: continue g = bfu.num2CG(gnum,n) if not bfu.call_u_conflicts(g, H): l.add(gnum) if bfu.call_u_equals(g, H): s.add(gnum) return list(l), s
def quadmerge(gl, H, cds): n = len(H) l = set() s = set() mask, ss = edgemask(gl, H, cds) s = s | ss ds = edgeds(mask) #pp = pprint.PrettyPrinter(indent=1) #pp.pprint(ds) for idx in ds: for gn in ds[idx]: if mask[idx] & gn: continue if skip_conflict(mask[idx], gn, cds): continue gnum = mask[idx] | gn if gnum in l or gnum in ss: continue g = bfu.num2CG(gnum, n) if not bfu.call_u_conflicts(g, H): l.add(gnum) if bfu.call_u_equals(g, H): s.add(gnum) return list(l), s
def withrates(s, H): n = len(H) d = {g: set() for g in s} for g in s: d[g] = getrates(bfu.num2CG(g, n), H) return d
def permuteAset(s, perm): n = len(perm) ns = set() for e in s: ns.add(bfu.g2num(permute(bfu.num2CG(e,n),perm))) return ns
def wrapper(fold,n=10,dens=0.1, urate=URATE): scipy.random.seed() rate = urate r = None s = set() counter = 0 while not s: scipy.random.seed() sst = 0.9 r = None while not r: r = lm.getAring(n, dens, sst, False, dist=DIST) print sst, sys.stdout.flush() if sst < 0.03: sst -= 0.001 else: sst -= 0.01 if sst < 0: sst = 0.02 #pprint.pprint(r['transition'].round(2),width=200) #d = zkl.load('leibnitz_nodes_'+str(n)+'_OCE_model_.zkl') #r = d[dens][fold] g = r['graph'] true_g2 = bfu.undersample(g, rate-1) data = lm.drawsamplesLG(r['transition'], samples=BURNIN+SAMPLESIZE*2, nstd=np.double(NOISE_STD)) data = data[:,BURNIN:] if np.max(data) > 1000.: pprint.pprint(r['transition'].round(2),width=200) #raise ValueError startTime = int(round(time.time() * 1000)) if EST=='pc': g2 = pc.dpc(data[:,::rate], pval=0.0001) elif EST=='svar': g2 = lm.data2graph(data[:,::rate]) if trv.density(g2) < 0.7: print gk.OCE(g2,true_g2) #s = examine_bidirected_flips(g2, depth=DEPTH) s = find_nearest_reachable(g2, max_depth=1) #s = trv.v2g22g1(g2, capsize=CAPSIZE, verbose=False) #s = trv.edge_backtrack2g1_directed(g2, capsize=CAPSIZE) #s = timeout(trv.v2g22g1, #s = timeout(trv.edge_backtrack2g1_directed, # args=(g2,CAPSIZE), # timeout_duration=1000, default=set()) print 'o', sys.stdout.flush() if -1 in s: s=set() endTime = int(round(time.time() * 1000)) #if counter > 3: # print 'not found' # return None counter += 1 print '' oce = [gk.OCE(bfu.num2CG(x,n),g) for x in s] cum_oce = [sum(x['directed'])+sum(x['bidirected']) for x in oce] idx = np.argmin(cum_oce) print "{:2}: {:8} : {:4} {:10} seconds".\ format(fold, round(dens,3), cum_oce[idx], round((endTime-startTime)/1000.,3)) #np.set_printoptions(formatter={'float': lambda x: format(x, '6.3f')+", "}) #pprint.pprint(r['transition'].round(2)) #np.set_printoptions() return {'gt':r, 'eq':s, 'OCE':oce[idx], 'tries_till_found': counter, 'estimate': g2, 'graphs_tried': counter, 'strength':sst+0.01, 'ms':endTime-startTime}
def wrapper(fold, n=10, dens=0.1, urate=URATE): scipy.random.seed() rate = urate r = None s = set() counter = 0 while not s: scipy.random.seed() sst = 0.9 r = None while not r: r = lm.getAring(n, dens, sst, False, dist=DIST) print sst, sys.stdout.flush() if sst < 0.03: sst -= 0.001 else: sst -= 0.01 if sst < 0: sst = 0.02 #pprint.pprint(r['transition'].round(2),width=200) #d = zkl.load('leibnitz_nodes_'+str(n)+'_OCE_model_.zkl') #r = d[dens][fold] g = r['graph'] true_g2 = bfu.undersample(g, rate - 1) data = lm.drawsamplesLG(r['transition'], samples=BURNIN + SAMPLESIZE * 2, nstd=np.double(NOISE_STD)) data = data[:, BURNIN:] if np.max(data) > 1000.: pprint.pprint(r['transition'].round(2), width=200) #raise ValueError startTime = int(round(time.time() * 1000)) if EST == 'pc': g2 = pc.dpc(data[:, ::rate], pval=0.0001) elif EST == 'svar': g2 = lm.data2graph(data[:, ::rate]) if trv.density(g2) < 0.7: print gk.OCE(g2, true_g2) #s = examine_bidirected_flips(g2, depth=DEPTH) s = find_nearest_reachable(g2, max_depth=1) #s = trv.v2g22g1(g2, capsize=CAPSIZE, verbose=False) #s = trv.edge_backtrack2g1_directed(g2, capsize=CAPSIZE) #s = timeout(trv.v2g22g1, #s = timeout(trv.edge_backtrack2g1_directed, # args=(g2,CAPSIZE), # timeout_duration=1000, default=set()) print 'o', sys.stdout.flush() if -1 in s: s = set() endTime = int(round(time.time() * 1000)) #if counter > 3: # print 'not found' # return None counter += 1 print '' oce = [gk.OCE(bfu.num2CG(x, n), g) for x in s] cum_oce = [sum(x['directed']) + sum(x['bidirected']) for x in oce] idx = np.argmin(cum_oce) print "{:2}: {:8} : {:4} {:10} seconds".\ format(fold, round(dens,3), cum_oce[idx], round((endTime-startTime)/1000.,3)) #np.set_printoptions(formatter={'float': lambda x: format(x, '6.3f')+", "}) #pprint.pprint(r['transition'].round(2)) #np.set_printoptions() return { 'gt': r, 'eq': s, 'OCE': oce[idx], 'tries_till_found': counter, 'estimate': g2, 'graphs_tried': counter, 'strength': sst + 0.01, 'ms': endTime - startTime }
def withrates(s,H): n = len(H) d = {g:set() for g in s} for g in s: d[g] = getrates(bfu.num2CG(g,n),H) return d