def g2pair(self,ni,nf,ei,ef1,ef2,ew,numFile,mapFile,weightFile): inobj = [] inobj.append(nm.mcut(f="%s:node"%(ef1),i=ei ).msetstr(a="flag",v=0)) inobj.append(nm.mcut(f="%s:node"%(ef2),i=ei ).msetstr(a="flag",v=0)) if nf : inobj.append(nm.mcut(f="%s:node"%(nf),i=ni ).msetstr(a="flag",v=1)) f = nm.mbest(i=inobj,k="node",s="flag",fr=0,size=1 ) # isolated nodes are set to the end of position in mapping file. # S= must start from 0 (but inside R vertex number will be added one) f <<= nm.mnumber(s="flag,node",a="num",S=0,o=mapFile) f.run() f = None f <<= nm.mcut(f=[ef1,ef2] , i=ei) f <<= nm.mjoin( k=ef1 , K="node" , m=mapFile ,f="num:num1") f <<= nm.mjoin( k=ef2 , K="node" , m=mapFile ,f="num:num2") f <<= nm.mcut(f="num1,num2") f <<= nm.mfsort(f="num1,num2") f <<= nm.msortf(f="num1%n,num2%n",nfno=True) f <<= nm.cmd("tr ',' ' ' " ) f <<= nm.mwrite(o=numFile) f.run() nodeSize=mrecount(i=mapFile) if ew: nm.mcut(f=ew,i=ei,o=weightFile).run() else: ew="weight" nm.msetstr(v=1,a=ew,i=ei).mcut(f=ew,o=weightFile).run() return nodeSize
def pair2tra(self, ei, ef1, ef2, traFile, mapFile1, mapFile2): f1 = None f1 <<= nm.mcut(f="{}:node1".format(self.ef1), i=self.ei) f1 <<= nm.mdelnull(f="node1") f1 <<= nm.muniq(k="node1") f1 <<= nm.mnumber(s="node1", a="num1", o=mapFile1) f2 = None f2 <<= nm.mcut(f="{}:node2".format(self.ef2), i=self.ei) f2 <<= nm.mdelnull(f="node2") f2 <<= nm.muniq(k="node2") f2 <<= nm.mnumber(s="node2", a="num2", o=mapFile2) f3 = None f3 <<= nm.mcut(f="{}:node1,{}:node2".format(self.ef1, self.ef2), i=self.ei) f3 <<= nm.mjoin(k="node1", m=f1, f="num1") f3 <<= nm.mjoin(k="node2", m=f2, f="num2") f3 <<= nm.mcut(f="num1,num2") f3 <<= nm.msortf(f="num1,num2%n") f3 <<= nm.mtra(k="num1", s="num2%n", f="num2") f3 <<= nm.msortf(f="num1%n") f3 <<= nm.mcut(f="num2", nfno=True, o=traFile) f3.run() os.system("cat " + traFile)
def edge2mtx(self, ei, itra, map1, map2): p1 = nm.mcut(f=self.ef1, i=ei) p1 <<= nm.muniq(k=self.ef1) p1 <<= nm.mdelnull(f=self.ef1) p1 <<= nm.mnumber(q=True, a="num1", S=1, o=map1) p2 = nm.mcut(f=self.ef2, i=ei) p2 <<= nm.muniq(k=self.ef2) p2 <<= nm.mdelnull(f=self.ef2) p2 <<= nm.mnumber(q=True, a="num2", S=1, o=map2) runp = None runp <<= nm.mcut(f=[self.ef1, self.ef2], i=ei) runp <<= nm.mjoin(k=self.ef1, m=p1, f="num1") runp <<= nm.mjoin(k=self.ef2, m=p2, f="num2") runp <<= nm.mcut(f="num1,num2") runp <<= nm.mtra(k="num1", f="num2") runp <<= nm.msortf(f="num1%n") runp <<= nm.mcut(f="num2", nfno=True) runp <<= nm.cmd("tr ',' ' '") runp <<= nm.mwrite(o=itra) #runp <<= nm.mcut(f="num2",nfno=True,o=wff1) runp.run()
def __convertToNumeric(self): wf1 = self.__tempW.file() wf2 = self.__tempW.file() wf3 = self.__tempW.file() self.mFile = self.__temp.file() nm.mcut(f="e1:node", i=self.__wfE, o=wf1).run() nm.mcut(f="e2:node", i=self.__wfE, o=wf2).run() mcmd = None if self.__wfN is not None: nm.mcut(f="n:node", i=self.__wfN, o=wf3).run() mcmd <<= nm.mcat(i=wf1 + "," + wf2 + "," + wf3, f="node") else: mcmd <<= nm.mcat(i=wf1 + "," + wf2, f="node") mcmd <<= nm.muniq(k="node") mcmd <<= nm.mnumber(q=True, a="id", o=self.mFile) mcmd.run() #エッジファイル変換・保存 self.eFile = self.__temp.file() mcmd = nm.mjoin(i=self.__wfE, m=self.mFile, f="id:id1", k="e1", K="node") mcmd <<= nm.mjoin(m=self.mFile, f="id:id2", k="e2", K="node") mcmd <<= nm.muniq(k="id1,id2") mcmd <<= nm.msortf(f="id1%n,id2%n") mcmd <<= nm.mcut(f="id1:e1,id2:e2,no:row_index", o=self.eFile) mcmd.run() #take内部用エッジファイル保存 self.eFileT = self.__temp.file() mcmd = nm.mcal(i=self.eFile, c="cat(\" \",$s{e1},$s{e2})", a="edge") mcmd <<= nm.mcut(nfno=True, f="edge", o=self.eFileT) mcmd.run() #ノードファイル変換・保存 self.nFile = self.__temp.file() if self.__wfN is not None: mcmd = nm.mjoin(i=self.__wfN, m=self.mFile, f="id", k="n", K="node") mcmd <<= nm.muniq(k="id") mcmd <<= nm.msortf(f="id%n") mcmd <<= nm.mcut(f="id:n,no:row_index", o=self.nFile) mcmd.run() else: #エッジよりノード生成 wf4 = self.__tempW.file() wf5 = self.__tempW.file() nm.mcut(i=self.eFile, f="e1:n", o=wf4).run() nm.mcut(i=self.eFile, f="e2:n", o=wf5).run() mcmd = nm.mcat(i="%s,%s" % (wf4, wf5)) mcmd <<= nm.muniq(k="n") mcmd <<= nm.msortf(f="n%n") mcmd <<= nm.mnumber(q=True, a="row_index", o=self.nFile) mcmd.run() #ワークファイル削除 self.__tempW.rm()
def __mkEdge(key, ef1, ef2, el, ec, ed, ev, ei, norm, mapFile, oFile): # mcal cat用のlabel項目の作成 label = [] if el: for nml in el: label.append("$s{" + nml + "}") evcdStr = [] #エッジの拡大率は固定 er = 10 if ev: evcdStr.append(ev + ":ev") if ec: evcdStr.append(ec + ":ec") if ed: evcdStr.append(ed + ":ed") f = None if el: f <<= nm.mcal(c='cat(\"_\",%s)' % (','.join(label)), a="##label", i=ei) else: f <<= nm.msetstr(v="", a="##label", i=ei) if len(evcdStr) == 0: f <<= nm.mcut(f="%s:key,%s:nam1,%s:nam2,##label:el" % (key, ef1, ef2)) else: f <<= nm.mcut(f="%s:key,%s:nam1,%s:nam2,##label:el,%s" % (key, ef1, ef2, ','.join(evcdStr))) if not ev: f <<= nm.msetstr(v="", a="ev") if not ed: f <<= nm.msetstr(v="", a="ed") if not ec: f <<= nm.msetstr(v="", a="ec") f <<= nm.mnullto(f="key", v="##NULL##") f <<= nm.mjoin(k="key", K="nam", m=mapFile, f="num:keyNum") f <<= nm.mjoin(k="nam1", K="nam", m=mapFile, f="num:num1,leaf:leaf1") f <<= nm.mjoin(k="nam2", K="nam", m=mapFile, f="num:num2,leaf:leaf2") if norm: f <<= nm.mnormalize(f="ev:ev2", c="range") f <<= nm.mcal(c='${ev2}*(%s-1)+1' % (er), a="evv") f <<= nm.mcut( f="key,nam1,nam2,keyNum,num1,num2,el,evv:ev,ed,ec,leaf1,leaf2", o=oFile) else: f <<= nm.mcut( f="key,nam1,nam2,keyNum,num1,num2,el,ev,ed,ec,leaf1,leaf2", o=oFile) f.run()
def __init__(self,db): self.db=db self.eArgs=None self.type =None self.minCnt=None self.minSup=None self.maxCnt=None self.maxSup=None self.minLen=None self.maxLen=None self.top =None self.skipTP=False #self.size =None self.pFile =None self.tFile =None self.msgoff = True self.temp=mtemp.Mtemp() self.db = db # 入力データベース self.file=self.temp.file() items=self.db.items # アイテムをシンボルから番号に変換する。 f = nm.mjoin(k=self.db.itemFN,K=items.itemFN,m=items.file,f=items.idFN,i=self.db.file) f <<= nm.mcut(f=self.db.idFN+","+items.idFN) f <<= nm.mtra(k=self.db.idFN,f=items.idFN) f <<= nm.mcut(f=items.idFN,nfno=True,o=self.file) f.run()
def convOrg(self,xxmap,xxout,ofile): xx1 = nm.mnumber(q=True,S=0,a="num",i=xxout) f = nm.mjoin(k="num",f="cls",m=xx1,i=xxmap) f <<= nm.mcut(f="node,cls", o=ofile) f.run()
def conv(self,iFile,idFN,itemFN,oFile,mapFile): temp=nu.Mtemp() xxtra=temp.file() # 入力ファイルのidがnilの場合は連番を生成して新たなid項目を作成する。 f0 = nm.mcut(f=itemFN+":##item",i=iFile) f0 <<= nm.mcount(k="##item",a="##freq") f0 <<= nm.mnumber(s="##freq%nr",a="##num",o=mapFile) f0.run() f1 = nm.mjoin(k=itemFN,K="##item",f="##num",m=mapFile,i=iFile,o=xxtra) f1.run() #f1 <<= nm.mtra(k=idFN,f="##num") #f1 <<= nm.mnumber(q=True,a="##traID") #f1 <<= nm.mcut(f="##num",nfno=True,o=oFile) extTake.mmaketra(i=xxtra,o=oFile,f="##num",k=idFN) #f1.run() # ##item,##freq%0nr,##num # b,4,0 # d,4,1 size=nu.mrecount(i=oFile,nfn=True) return size
def toNum(): for size in ["all"]: iFile = "%s/online_all.csv" % datPath oFile1 = "%s/onlineT_all.csv" % datPath # data for Take.core oFile2 = "%s/onlineO_all.basket" % datPath # data for Orange oFile3 = "%s/onlineM_all.csv" % datPath # data for Take f = None f <<= nm.mcut(f="InvoiceNo,StockCode", i=iFile) f <<= nm.muniq(k="InvoiceNo,StockCode") f <<= nm.mfldname(q=True, o=oFile3) f.run(msg=debug) st = None st <<= nm.mcut(f="StockCode", i=iFile) st <<= nm.muniq(k="StockCode") st <<= nm.mnumber(s="StockCode", a="num") f = None f <<= nm.mjoin(k="StockCode", m=st, f="num", i=iFile) f <<= nm.mcut(f="InvoiceNo,num:StockCode") f <<= nm.mtra(k="InvoiceNo", f="StockCode") f <<= nm.mcut(f="StockCode", nfno=True, o=oFile1) f.run(msg=debug) os.system("tr ' ' ',' <%s >%s" % (oFile1, oFile2))
def __mkMap(key, nf, ni, ef1, ef2, ei, oFile): # leaf nodeの構築 infL = [ nm.mcommon(k=ef1, K=key, m=ei, r=True, i=ei).mcut(f="%s:nam" % (ef1)), nm.mcommon(k=ef2, K=key, m=ei, r=True, i=ei).mcut(f="%s:nam" % (ef2)) ] if ni: infL.append( nm.mcommon(k=nf, K=key, m=ei, r=True, i=ni).mcut(f="%s:nam" % (nf))) xleaf = nm.muniq(i=infL, k="nam") xleaf <<= nm.msetstr(v=1, a="leaf") if ni: inp = [ nm.mcut(f="%s:nam" % (nf), i=ni), nm.mcut(f="%s:nam" % (key), i=ni) ] else: inp = [ nm.mcut(f="%s:nam" % (ef1), i=ei), nm.mcut(f="%s:nam" % (ef2), i=ei), nm.mcut(f="%s:nam" % (key), i=ei) ] f = None f <<= nm.muniq(k="nam", i=inp) f <<= nm.mjoin(k="nam", m=xleaf, f="leaf", n=True) # nullは最初に来るはずなので、mcalでなくmnumberでもnullを0に採番できるはずだが念のために f <<= nm.mcal(c='if(isnull($s{nam}),0,line()+1)', a="num") f <<= nm.mnullto(f="nam", v="##NULL##", o=oFile) f.run()
def __init__(self, db, outtf=True): self.size = None self.pFile = None self.tFile = None self.temp = mtemp.Mtemp() self.db = db # 入力データベース self.file = self.temp.file() self.outtf = outtf self.weightFile = {} self.posWeight = {} self.sigma = {} self.msgoff = True items = self.db.items for cName, posSize in db.clsNameRecSize.items(): self.weightFile[cName] = self.temp.file() self.posWeight[cName] = self.calOmega(posSize) f = nm.mcut(nfno=True, f=self.db.clsFN, i=self.db.cFile) f <<= nm.mchgstr(nfn=True, f=0, O=-1, o=self.weightFile[cName], c="%s:%s" % (cName, self.posWeight[cName])) f.run() # アイテムをシンボルから番号に変換する。 f = nm.mjoin(k=self.db.itemFN, K=items.itemFN, i=self.db.file, m=items.file, f=items.idFN) f <<= nm.mcut(f=self.db.idFN + "," + items.idFN) f <<= nm.mtra(k=self.db.idFN, f=items.idFN) f <<= nm.mcut(f=items.idFN, nfno=True, o=self.file) f.run()
def g2pair(self,ni,nf,ei,ef1,ef2,ew,numFile,mapFile,weightFile): #MCMD::msgLog("converting graph files into a pair of numbered nodes ...") #wf=MCMD::Mtemp.new #wf1=wf.file #wf2=wf.file #wf3=wf.file allinObj =[] wf1 = nm.mcut(f="%s:node"%(ef1),i=ei ).msetstr(v=0,a="flag") wf2 = nm.mcut(f="%s:node"%(ef2),i=ei ).msetstr(v=0,a="flag") f = None if nf: f <<= nm.mcut(i=[wf1,wf2,nm.mcut(f=nf+":node",i=ni).msetstr(v=1,a="flag")],f="node,flag") f <<= nm.mbest(k="node" , s="flag" , fr=0 ,size=1) else: f <<= nm.mcut(i=[wf1,wf2],f="node,flag") f <<= nm.muniq(k="node") f <<= nm.mnumber(s="flag,node",a="num",S=0,o=mapFile) f.run() f = None f <<= nm.mcut(f=[ef1,ef2],i=ei) f <<= nm.mjoin(k=ef1 , K="node",m=mapFile , f="num:num1") f <<= nm.mjoin(k=ef2 , K="node",m=mapFile , f="num:num2") f <<= nm.mcut(f="num1,num2") f <<= nm.mfsort(f="num1,num2") f <<= nm.msortf(f="num1%n,num2%n",nfno=True) f <<= nm.cmd("tr ',' ' ' ") f <<= nm.mwrite(o=numFile) f.run() if ew : nm.mcut(f=ew,i=ei,o=weightFile).run() else: ew="weight" nm.msetstr(v=1 , a=ew ,i=ei).mcut(f=ew,o=weightFile).run() nodeSize=mrecount(i=mapFile) return nodeSize
def repTaxo(self, taxonomy): #@taxonomy=taxonomy #replaceの場合はtaxonomyを登録しない self.items.repTaxo(taxonomy) # アイテムクラスをtaxonomyで置換する tFile = taxonomy.file itemFN = taxonomy.itemFN taxoFN = taxonomy.taxoFN tf = nu.Mtemp() xx1 = tf.file() cpara = "%s,%s:%s" % (self.idFN, taxoFN, self.itemFN) kpara = "%s,%s" % (self.idFN, self.itemFN) nm.mjoin(i=self.file, k=self.itemFN, K=itemFN, f=taxoFN, m=tFile).mcut(f=cpara).muniq(k=kpara, o=xx1).run() self.file = self.temp.file() shutil.move(xx1, self.file)
def convRsl(self, ifile, ofile, map1, map2, logDir=None): # 上記iterationで収束したマイクロクラスタグラフを元の節点文字列に直して出力する #MCMD::msgLog("converting the numbered nodes into original name ...") f = None f <<= nm.mcut(nfni=True, f="0:tra", i=ifile) f <<= nm.msed(f="tra", c=' $', v="") f <<= nm.mnumber(q=True, S=1, a="num1") f <<= nm.mtra(r=True, f="tra:num2") f <<= nm.mjoin(k="num2", m=map2, f=self.ef2) f <<= nm.mjoin(k="num1", m=map1, f=self.ef1) f <<= nm.msortf(f="num1%n,num2%n") f <<= nm.mcut(f=[self.ef1, self.ef2]) if logDir: f <<= nm.mfldname(q=True, o="{}/#{ofile}".format(logDir, ofile)) else: f <<= nm.mfldname(q=True, o=ofile) f.run()
def g2pair(self, ni, nf, ei, ef1, ef2, ipair, mapFile): dlist = [ nm.mcut(f=ef1 + ":node", i=ei), nm.mcut(f=ef2 + ":node", i=ei) ] if (ni): dlist.append(nm.mcut(f=nf + ":node", i=ni)) fmap = None fmap <<= nm.mcut(i=dlist, f="node") fmap <<= nm.muniq(k="node") fmap <<= nm.mnumber(q=True, a="num", o=mapFile) f = nm.mcut(f=[ef1, ef2], i=ei) f <<= nm.mjoin(k=ef1, K="node", m=fmap, f="num:num1") f <<= nm.mjoin(k=ef2, K="node", m=fmap, f="num:num2") f <<= nm.mcut(f="num1,num2") f <<= nm.mfsort(f="num1,num2") f <<= nm.msortf(f="num1%n,num2%n", nfno=True, o=ipair) f.run()
def calRelative(iFile, oFile): mcut = None mcut <<= nm.mselstr(f="method", v="mcut", i="methods.csv") f = None f <<= nm.mnumber(q=True, a="id", i=iFile) f <<= nm.mjoin(k="dataSize", m=mcut, f="mean:base") f <<= nm.mcal(c='round(${mean}/${base},0.1)', a="score") f <<= nm.m2cross(k="method", s="dataSize", f="score") f <<= nm.msortf(f="id%n") f <<= nm.mcut(f="method,10000:small,1000000:middle,100000000:large") f <<= nm.mfldname(q=True, o=oFile) f.run()
def convSim(self, ifile, ofile, map1, logDir): f = None f <<= nm.mcut(nfni=True, f="0:tra", i=ifile) f <<= nm.msed(f="tra", c=' $', v="") f <<= nm.mnumber(q=True, S=1, a="num1") f <<= nm.mtra(r=True, f="tra:num11") f <<= nm.mnumber(q=True, S=1, a="order") f <<= nm.mcal(c='${num11}+1', a="num1") f <<= nm.mjoin(k="num1", m=map1, f=self.ef1) f <<= nm.mtra(k="num0", s="order%n,num1%n", f=self.ef1) f <<= nm.mcut(f=self.ef1, o="{}/{}".format(logDir, ofile)) f.run()
def g2pair(self, ni, nf, ei, ef1, ef2, numFile, mapFile): #MCMD::msgLog("converting graph files into a pair of numbered nodes ...") #wf=MCMD::Mtemp.new #wf1=wf.file #wf2=wf.file #wf3=wf.file inobj = [] inobj.append(nm.mcut(f="%s:node" % (ef1), i=ei).msetstr(a="flag", v=0)) inobj.append(nm.mcut(f="%s:node" % (ef2), i=ei).msetstr(a="flag", v=0)) if nf: inobj.append( nm.mcut(f="%s:node" % (nf), i=ni).msetstr(a="flag", v=1)) f = nm.mbest(i=inobj, k="node", s="flag", fr=0, size=1) # isolated nodes are set to the end of position in mapping file. # S= must start from 0 (but inside R vertex number will be added one) f <<= nm.mnumber(s="flag,node", a="num", S=0, o=mapFile) f.run() f = None f <<= nm.mcut(f=[ef1, ef2], i=ei) f <<= nm.mjoin(k=ef1, K="node", m=mapFile, f="num:num1") f <<= nm.mjoin(k=ef2, K="node", m=mapFile, f="num:num2") f <<= nm.mcut(f="num1,num2") #f << "mfsort f=num1,num2 |" f <<= nm.msortf(f="num1%n,num2%n", nfno=True) f <<= nm.cmd("tr ',' ' ' ") f <<= nm.mwrite(o=numFile) f.run() nodeSize = mrecount(i=mapFile) return nodeSize
def readCSV(iParams): iFile = iParams["iFile"] sidF = iParams["sid"] eidF = iParams["time"] itemF = iParams["item"] temp = Mtemp() xxdatPath = temp.file() mkDir(xxdatPath) # classファイルの処理 if "cFile" in iParams: cFile = iParams["cFile"] csidF = iParams["csid"] classF = iParams["cNames"] f = None f <<= nm.mcut(f="%s:sid,%s:eid,%s:item" % (sidF, eidF, itemF), i=iFile) f <<= nm.mdelnull(f="sid,eid,item") f <<= nm.muniq(k="sid,eid,item") f <<= nm.mjoin(k="sid", K=csidF, m=cFile, f="%s:class" % (classF)) f <<= nm.msep(s="sid,eid%n,item", d="%s/${class}" % (xxdatPath), p=True) f.run() classNames = glob.glob("%s/*" % (xxdatPath)) classNames = [os.path.basename(path) for path in classNames] else: f = None f <<= nm.mcut(f="%s:sid,%s:eid,%s:item" % (sidF, eidF, itemF), i=iFile) f <<= nm.mdelnull(f="sid,eid,item") f <<= nm.muniq(k="sid,eid,item") f <<= nm.msortf(f="sid,eid%n,item", o="%s/single" % (xxdatPath)) f.run() classNames = ["single"] datas = {} for name in classNames: dataFile = "%s/%s" % (xxdatPath, name) datas[name] = _readCSV_sub(dataFile) return datas
def runmain(self,edgeFile): import re baseName = re.sub('\.edge$',"",edgeFile) name = re.sub('^.*\/',"",baseName) if self.ni : nodeFile=self.ni else : nodeFile=re.sub('\.edge$',".node",edgeFile) # convert the original graph to one igraph can handle temp=Mtemp() xxnum = temp.file() xxmap = temp.file() xxout = temp.file() xxscp = temp.file() xxweight=temp.file() nodeSize=self.g2pair( nodeFile,self.nf,edgeFile,self.ef1,self.ef2,self.ew,xxnum,xxmap,xxweight ) # generate R script, and run self.genRscript( self.directed,self.norm,self.mode, xxnum, xxweight, self.ew,nodeSize, xxout, xxscp ) if self.verbose : os.system("R --vanilla -q < %s"%(xxscp)) else: os.system("R --vanilla -q --slave < %s 2>/dev/null "%(xxscp)) f = None f <<= nm.mnumber(q=True,S=0,a="num",i=xxout) f <<= nm.mjoin(k="num",f="node",m=xxmap ) outf = self.oPath + "/" +name + ".csv" if self.nf : f <<= nm.mcut(f="node:%s,degree,cc,components,betweenness,closeness,page_rank"%(self.nf),o=outf) else: f <<= nm.mcut(f="node,degree,cc,components,betweenness,closeness,page_rank",o=outf) f.run()
def mnest2tree(ei, ef, k, ni=None, nf=None, ev=None, no=None, eo=None): # paracheck追加 efs = ef.split(",") ef1 = efs[0] ef2 = efs[1] f = nm.mcut(f="%s:#orgKey,%s:#orgEf1,%s:#orgEf2" % (k, ef1, ef2), i=ei) temp = mtemp.Mtemp() of = temp.file() with _nu.mcsvout(o=of, f="#orgKey,#orgEf1,#orgEf2,#ef1,#ef2") as oCSV: for flds in f: orgKey = flds[0] orgEf1 = flds[1] orgEf2 = flds[2] oCSV.write([orgKey, orgEf1, orgEf2, orgKey, orgEf1]) oCSV.write([orgKey, orgEf1, orgEf2, orgKey, orgEf2]) f = None f <<= nm.mjoin(k="#orgKey,#orgEf1,#orgEf2", K=[k, ef1, ef2], m=ei, i=of) # 全項目join if ev: f <<= nm.mavg(k="#ef1,#ef2", f=ev) else: f <<= nm.muniq(k="#ef1,#ef2") f <<= nm.mcut(r=True, f="#orgKey,#orgEf1,#orgEf2") f <<= nm.mfldname(f="#ef1:%s,#ef2:%s" % (ef1, ef2), o=eo) f.run() if ni: head = nu.mheader(i=ni) fldnames = [s for s in head if s != nf] commas = ',' * (len(fldnames) - 1) f0 = None f0 <<= nm.mcut(f="%s:%s" % (ef1, nf), i=eo) f0 <<= nm.muniq(k=nf) f0 <<= nm.mcommon(k=nf, m=ni, r=True) f0 <<= nm.msetstr(v=commas, a=fldnames) f = nm.mcut(f=k, r=True, i=[ni, f0]) f <<= nm.msetstr(v="", a=k, o=no) f.run()
def tra2tbl(iFile, oFile, tidFld, itemFld, null=0, dummy=True, aggFld=None, aggStat=None, klassFld=None): klass = None if klassFld is not None and klassFld != "": klass <<= nm.mcut(f=tidFld + "," + klassFld, i=iFile) klass <<= nm.muniq(k=tidFld) f = None # aggFldが指定されていればセル項目を集計 if aggFld is not None and aggFld != "": f <<= nm.mcut(f=tidFld + "," + itemFld + "," + aggFld, i=iFile) f <<= nm.mstats(k=tidFld, f="%s:_cell" % (aggFld), c=aggStat) # aggFldが指定されていなければカウント else: f <<= nm.mcut(f=tidFld + "," + itemFld, i=iFile) # アイテムが出現したかどうか if dummy: f <<= nm.muniq(k=tidFld + "," + itemFld) f <<= nm.msetstr(v=1, a="_cell") # アイテムが何件出現したかどうか else: f <<= nm.mcount(k=tidFld + "," + itemFld, a="_cell") # 横展開 f <<= nm.m2cross(k=tidFld, s=itemFld, f="_cell") # クラス項目が指定されていれば結合する if klassFld is not None and klassFld != "": f <<= nm.mjoin(k=tidFld, m=klass, f=klassFld) # null値を一斉に置換する f <<= nm.mnullto(f="*", v=null, o=oFile) f.run(msg="on")
def __init__(self, db, outtf=True): self.size = None self.temp = mtemp.Mtemp() self.db = db # 入力データベース self.file = self.temp.file() items = self.db.items self.outtf = outtf self.top = None self.msgoff = True # アイテムをシンボルから番号に変換する。 f = nm.mjoin(k=self.db.itemFN, K=items.itemFN, m=items.file, f=items.idFN, i=self.db.file) f <<= nm.mcut(f=self.db.idFN + "," + self.db.timeFN + "," + items.idFN) f <<= nm.mtra(k=self.db.idFN, s=self.db.timeFN + "%n", f=items.idFN) f <<= nm.mcut(f=items.idFN, nfno=True, o=self.file) f.run()
def repTaxo(self, taxonomy): #@taxonomy=taxonomy self.items.repTaxo(taxonomy) # アイテムクラスをtaxonomyで置換する tFile = taxonomy.file itemFN = taxonomy.itemFN taxoFN = taxonomy.taxoFN tf = mtemp.Mtemp() xx1 = tf.file() f = None f <<= nm.mjoin(k=self.itemFN, K=itemFN, f=taxoFN, m=tFile, i=self.file) f <<= nm.mcut(f=self.idFN + "," + self.timeFN + "," + taxoFN + ":" + self.itemFN) f <<= nm.msortf(f=self.idFN + "," + self.timeFN + "," + self.itemFN, o=xx1) f.run() self.file = self.temp.file() shutil.move(xx1, self.file)
def __init__(self,db,outtf=True): self.size = None self.msgoff = True self.temp = nu.Mtemp() self.db = db # 入力データベース self.file = self.temp.file() self.outtf = outtf items = self.db.items # 重みファイルの作成 # pos,negのTransactionオブジェクトに対してLCMが扱う整数アイテムによるトランザクションファイルを生成する。 # この時、pos,negを併合して一つのファイルとして作成され(@wNumTraFile)、 # 重みファイル(@weightFile[クラス])の作成は以下の通り。 # 1.対象クラスをpos、その他のクラスをnegとする。 # 2. negの重みは-1に設定し、posの重みはcalOmegaで計算した値。 # 3.@wNumTraFileの各行のクラスに対応した重みデータを出力する(1項目のみのデータ)。 self.weightFile = {} self.posWeight = {} self.sigma = {} for cName,posSize in db.clsNameRecSize.items(): self.weightFile[cName] = self.temp.file() self.posWeight[cName] = self.calOmega(posSize) cpara = "%s:%s"%(cName,self.posWeight[cName]) nm.mcut(nfno=True,f=self.db.clsFN,i=self.db.cFile).mchgstr(nfn=True,f=0,O=-1,o=self.weightFile[cName],c=cpara).run() # アイテムをシンボルから番号に変換する。 f=None f <<= nm.mjoin(k=self.db.itemFN,K=items.itemFN,m=items.file,f=items.idFN,i=self.db.file) f <<= nm.mcut(f=self.db.idFN+","+self.db.timeFN+","+items.idFN) f <<= nm.msortf(f=self.db.idFN+","+self.db.timeFN+"%n") f <<= nm.mtra(k=self.db.idFN,f=items.idFN) f <<= nm.mcut(f=items.idFN,nfno=True,o=self.file) f.run()
import nysol.mcmd as nm os.environ["KG_VerboseLevel"] = "3" debug = "on" iPath = "./DATA" oPath = "./OUTPUT/apriori" os.system("mkdir -p %s" % oPath) iFile = "%s/price_large.csv" % (iPath) topix = "%s/index.csv" % (iPath) # make a transaction data, which date as a transaction and tickerID as an item tra = None tra <<= nm.mcut(f="id,date,c", i=iFile) tra <<= nm.mjoin(k="date", m=topix, f="i") tra <<= nm.mslide(k="id", s="date", f="date:date2,c:c2,i:i2") tra <<= nm.mcal(c="${c2}/${c}-${i2}/${i}", a="ret") tra <<= nm.mselnum(f="ret", c="[0.05,0.1]") tra <<= nm.mcut(f="id,date2:date,ret") # frequency of one item freq = None freq <<= nm.mcut(f="id", i=tra) freq <<= nm.mcount(k="id", a="freq") freq <<= nm.mselnum(f="freq", c="[5,]") # total number of transactions total = None total <<= nm.mcut(f="date", i=tra) total <<= nm.muniq(k="date")
def run(self): from datetime import datetime t = datetime.now() temp=nu.Mtemp() xxsspcin=temp.file() xxmap=temp.file() # traファイルの変換とマップファイルの作成 if self.num : total = self.convN(self.iFile,self.idFN,self.itemFN,xxsspcin,xxmap) else: total = self.conv(self.iFile,self.idFN,self.itemFN,xxsspcin,xxmap) # system "head xxsspcin" # 3 5 0 2 # 4 1 2 # 0 2 3 1 # 1 0 2 # 3 4 0 1 # system "head xxmap" # ##item,##freq%0nr,##num # b,4,0 # d,4,1 # f,4,2 minSupp = int(total*self.minSupPrb) if self.minSupPrb else self.minSupCnt # sspc用simの文字列 if self.sim : if self.sim=="J": sspcSim="R" elif self.sim=="P": sspcSim="P" elif self.sim=="C": sspcSim="i" # sim=省略時はRでth=0とする(sim制約なし) else: sspcSim="R" self.th=0 ############ 列挙本体 ############ xxsspcout=temp.file() tpstr = sspcSim+"ft_" if self.msgoff else sspcSim+"ft" extTake.sspc(type=tpstr,TT=minSupp,i=xxsspcin,th=self.th,o=xxsspcout) ################################## xxtmmp=temp.file() f = nm.mread(i=xxsspcout) f <<= nm.cmd("tr ' ()' ','") f <<= nm.mcut(f="1:i1,2:i2,0:frequency,4:sim",nfni=True) if self.num : f <<= nm.mfldname(f="i1:node1,i2:node2") if self.sim!="C": f <<= nm.mfsort(f="node1,node2") f <<= nm.mjoin(k="node1",K="##item",m=xxmap,f="##freq:frequency1") f <<= nm.mjoin(k="node2",K="##item",m=xxmap,f="##freq:frequency2") else: f <<= nm.mjoin(k="i1",K="##num",m=xxmap,f="##item:node1,##freq:frequency1") f <<= nm.mjoin(k="i2",K="##num",m=xxmap,f="##item:node2,##freq:frequency2") if self.sim!="C": f <<= nm.mcut(f="i1,i2,frequency,sim,node1,node2,frequency1,frequency2,node1:node1x,node2:node2x") f <<= nm.mfsort(f="node1x,node2x") f <<= nm.mcal(c='if($s{node1}==$s{node1x},$s{frequency1},$s{frequency2})',a="freq1") f <<= nm.mcal(c='if($s{node2}==$s{node2x},$s{frequency2},$s{frequency1})',a="freq2") f <<= nm.mcut(f="i1,i2,frequency,sim,node1x:node1,node2x:node2,freq1:frequency1,freq2:frequency2") f <<= nm.msetstr(v=total,a="total") f <<= nm.mcal(c='${frequency}/${frequency1}',a="confidence") f <<= nm.mcal(c='${frequency}/${total}',a="support") f <<= nm.mcal(c='${frequency}/(${frequency1}+${frequency2}-${frequency})',a="jaccard") f <<= nm.mcal(c='(${frequency}*${total})/((${frequency1}*${frequency2}))',a="lift") f <<= nm.mcal(c='(ln(${frequency})+ln(${total})-ln(${frequency1})-ln(${frequency2}))/(ln(${total})-ln(${frequency}))',a="PMI") f <<= nm.mcut(f="node1,node2,frequency,frequency1,frequency2,total,support,confidence,lift,jaccard,PMI") f <<= nm.msortf(f="node1,node2",o=self.oeFile) f.run() if self.onFile: f4 = nm.mcut(f=self.itemFN+":node",i=self.iFile) f4 <<= nm.mcount(k="node",a="frequency") if self.node_support : minstr = "[%s,]"%(minSupp) f4 <<= nm.mselnum(f="frequency",c=minstr) f4 <<= nm.msetstr(v=total,a="total") f4 <<= nm.mcal(c='${frequency}/${total}',a="support") f4 <<= nm.mcut(f="node,support,frequency,total",o=self.onFile) f4.run() procTime=datetime.now()-t # ログファイル出力 if self.logFile : kv=[["key","value"]] for k,v in self.args.items(): kv.append([k,str(v)]) kv.append(["time",str(procTime)]) nm.writecsv(i=kv,o=self.logFile).run()
def msankey(i, o, v, f, title="", h=500, w=960, nl=False, T=None): # f= 2 fld if type(f) is str: ff = f.split(',') elif type(f) is list: ff = f else: raise TypeError("f= unsupport " + str(type(k))) if len(ff) < 2: raise TypeError("f= takes just two field names") if T != None: import re os.environ["KG_TmpPath"] = re.sub(r'/$', "", T) if h == None: h = 500 if w == None: w = 960 if title == None: title = "" tempW = mtemp.Mtemp() nodef = tempW.file() edgef = tempW.file() ef1 = ff[0] ef2 = ff[1] ev = v iFile = i oFile = o f0 = nm.mcut(i=iFile, f="%s:nodes" % (ef1)) f1 = nm.mcut(i=iFile, f="%s:nodes" % (ef2)) f2 = None f2 <<= nm.muniq(i=[f0, f1], k="nodes") f2 <<= nm.mnumber(s="nodes", a="num", o=nodef) f2.run() f3 = None f3 <<= nm.mcut(f="%s:nodes1,%s:nodes2,%s" % (ef1, ef2, ev), i=iFile) f3 <<= nm.mjoin(k="nodes1", K="nodes", m=nodef, f="num:num1") f3 <<= nm.mjoin(k="nodes2", K="nodes", m=nodef, f="num:num2") f3 <<= nm.mcut(f="num1,num2,%s" % (ev)) f3 <<= nm.msortf(f="num1%n,num2%n", o=edgef) f3.run() wk = [] nodeL = [] for flds in nm.readcsv(nodef).getline(otype='dict'): nodeL.append({"name": flds['nodes']}) nodes = json.JSONEncoder().encode(nodeL) linkL = [] for flds in nm.readcsv(edgef).getline(otype='dict', dtype={ "num1": "int", "num2": "int", ev: "int" }): linkL.append({ "source": flds["num1"], "target": flds["num2"], "value": flds[ev] }) links = json.JSONEncoder().encode(linkL) nolabel = "" if nl: nolabel = "font-size: 0px;" html = sys.stdout if not oFile == None: html = open(oFile, "w") outTemplate = ''' <!DOCTYPE html> <html class="ocks-org do-not-copy"> <meta charset="utf-8"> <title>{title}</title> <style> body {{ font: 10px sans-serif; }} svg {{ padding: 10px 0 0 10px; }} .arc {{ stroke: #fff; }} #tooltip {{ position: absolute; width: 150px; height: auto; padding: 10px; background-color: white; -webkit-border-radius: 10px; -moz-border-radius: 10px; border-radius: 10px; -webkit-box-shadow: 4px 4px 10px rgba(0,0,0,0.4); -moz-box-shadow: 4px 4px 10px rgba(0,0,0,0.4); box-shadow: 4px 4px 10px rgba(0,0,0,0.4); pointer-events: none; }} #tooltip.hidden {{ display: none; }} #tooltip p {{ margin: 0; font-family: sans-serif; font-size: 10px; line-height: 14px; }} #chart {{ height: 500px; }} .node rect {{ cursor: move; fill-opacity: .9; shape-rendering: crispEdges; }} .node text {{ pointer-events: none; text-shadow: 0 1px 0 #fff; {nolabel} }} .link {{ fill: none; stroke: #000; stroke-opacity: .2; }} .link:hover {{ stroke-opacity: .5; }} </style> <body> <h1>{title}</h1> <p id="chart"> <script> {d3js_str} d3.sankey = function() {{ var sankey = {{}}, nodeWidth = 24, nodePadding = 8, size = [1, 1], nodes = [], links = []; sankey.nodeWidth = function(_) {{ if (!arguments.length) return nodeWidth; nodeWidth = +_; return sankey; }}; sankey.nodePadding = function(_) {{ if (!arguments.length) return nodePadding; nodePadding = +_; return sankey; }}; sankey.nodes = function(_) {{ if (!arguments.length) return nodes; nodes = _; return sankey; }}; sankey.links = function(_) {{ if (!arguments.length) return links; links = _; return sankey; }}; sankey.size = function(_) {{ if (!arguments.length) return size; size = _; return sankey; }}; sankey.layout = function(iterations){{ computeNodeLinks(); computeNodeValues(); computeNodeBreadths(); computeNodeDepths(iterations); computeLinkDepths(); return sankey; }}; sankey.relayout = function() {{ computeLinkDepths(); return sankey; }}; sankey.link = function() {{ var curvature = .5; function link(d) {{ var x0 = d.source.x + d.source.dx, x1 = d.target.x, xi = d3.interpolateNumber(x0, x1), x2 = xi(curvature), x3 = xi(1 - curvature), y0 = d.source.y + d.sy + d.dy / 2, y1 = d.target.y + d.ty + d.dy / 2; return "M" + x0 + "," + y0 + "C" + x2 + "," + y0 + " " + x3 + "," + y1 + " " + x1 + "," + y1; }} link.curvature = function(_) {{ if (!arguments.length) return curvature; curvature = +_; return link; }}; return link; }}; // Populate the sourceLinks and targetLinks for each node. // Also, if the source and target are not objects, assume they are indices. function computeNodeLinks() {{ nodes.forEach(function(node) {{ node.sourceLinks = []; node.targetLinks = []; }}); links.forEach(function(link) {{ var source = link.source, target = link.target; if (typeof source === "number") source = link.source = nodes[link.source]; if (typeof target === "number") target = link.target = nodes[link.target]; source.sourceLinks.push(link); target.targetLinks.push(link); }}); }} // Compute the value (size) of each node by summing the associated links. function computeNodeValues() {{ nodes.forEach( function(node) {{ node.value = Math.max(d3.sum(node.sourceLinks, value),d3.sum(node.targetLinks, value)); }} ); }} // Iteratively assign the breadth (x-position) for each node. // Nodes are assigned the maximum breadth of incoming neighbors plus one; // nodes with no incoming links are assigned breadth zero, while // nodes with no outgoing links are assigned the maximum breadth. function computeNodeBreadths() {{ var remainingNodes = nodes, nextNodes, x = 0; while (remainingNodes.length) {{ nextNodes = []; remainingNodes.forEach(function(node) {{ node.x = x; node.dx = nodeWidth; node.sourceLinks.forEach(function(link) {{ nextNodes.push(link.target); }}); }}); remainingNodes = nextNodes; ++x; }} // moveSinksRight(x); scaleNodeBreadths((width - nodeWidth) / (x - 1)); }} function moveSourcesRight() {{ nodes.forEach(function(node) {{ if (!node.targetLinks.length) {{ node.x = d3.min(node.sourceLinks, function(d) {{ return d.target.x; }} ) - 1; }} }}); }} function moveSinksRight(x) {{ nodes.forEach(function(node) {{ if (!node.sourceLinks.length) {{ node.x = x - 1; }} }}); }} function scaleNodeBreadths(kx) {{ nodes.forEach(function(node) {{ node.x *= kx; }}); }} function computeNodeDepths(iterations) {{ var nodesByBreadth = d3.nest() .key(function(d) {{ return d.x; }}) .sortKeys(d3.ascending) .entries(nodes) .map(function(d) {{ return d.values; }}); // initializeNodeDepth(); resolveCollisions(); for (var alpha = 1; iterations > 0; --iterations){{ relaxRightToLeft(alpha *= .99); resolveCollisions(); relaxLeftToRight(alpha); resolveCollisions(); }} function initializeNodeDepth() {{ var ky = d3.min(nodesByBreadth, function(nodes) {{ return (size[1] - (nodes.length - 1) * nodePadding) / d3.sum(nodes, value); }}); nodesByBreadth.forEach(function(nodes) {{ nodes.forEach(function(node, i) {{ node.y = i; node.dy = node.value * ky; }}); }}); links.forEach(function(link) {{ link.dy = link.value * ky; }}); }} function relaxLeftToRight(alpha) {{ nodesByBreadth.forEach(function(nodes, breadth) {{ nodes.forEach(function(node) {{ if (node.targetLinks.length) {{ var y = d3.sum(node.targetLinks, weightedSource) / d3.sum(node.targetLinks, value); node.y += (y - center(node)) * alpha; }} }}); }}); function weightedSource(link) {{ return center(link.source) * link.value; }} }} function relaxRightToLeft(alpha) {{ nodesByBreadth.slice().reverse().forEach(function(nodes){{ nodes.forEach(function(node) {{ if (node.sourceLinks.length) {{ var y = d3.sum(node.sourceLinks, weightedTarget) / d3.sum(node.sourceLinks, value); node.y += (y - center(node)) * alpha; }} }}); }}); function weightedTarget(link) {{ return center(link.target) * link.value; }} }} function resolveCollisions() {{ nodesByBreadth.forEach(function(nodes) {{ var node, dy, y0 = 0, n = nodes.length, i; // Push any overlapping nodes down. nodes.sort(ascendingDepth); for (i = 0; i < n; ++i) {{ node = nodes[i]; dy = y0 - node.y; if (dy > 0) node.y += dy; y0 = node.y + node.dy + nodePadding; }} // If the bottommost node goes outside the bounds, push it back up. dy = y0 - nodePadding - size[1]; if (dy > 0) {{ y0 = node.y -= dy; // Push any overlapping nodes back up. for (i = n - 2; i >= 0; --i) {{ node = nodes[i]; dy = node.y + node.dy + nodePadding - y0; if (dy > 0) node.y -= dy; y0 = node.y; }} }} }}); }} function ascendingDepth(a, b) {{ return a.y - b.y; }} }} function computeLinkDepths() {{ nodes.forEach(function(node) {{ node.sourceLinks.sort(ascendingTargetDepth); node.targetLinks.sort(ascendingSourceDepth); }}); nodes.forEach(function(node) {{ var sy = 0, ty = 0; node.sourceLinks.forEach(function(link) {{ link.sy = sy; sy += link.dy; }}); node.targetLinks.forEach(function(link) {{ link.ty = ty; ty += link.dy; }}); }}); function ascendingSourceDepth(a, b) {{ return a.source.y - b.source.y; }} function ascendingTargetDepth(a, b) {{ return a.target.y - b.target.y; }} }} function center(node){{ return node.y + node.dy / 2; }} function value(link) {{ return link.value; }} return sankey; }}; </script> <script> var margin = {{top: 1, right: 1, bottom: 6, left: 1}}, width = {width} - margin.left - margin.right, height = {height} - margin.top - margin.bottom; var formatNumber = d3.format(",.0f"), format = function(d) {{ return formatNumber(d) + " TWh"; }}, color = d3.scale.category20(); var svg = d3.select("#chart").append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); var sankey = d3.sankey() .nodeWidth(15) .nodePadding(10) .size([width, height]); var path = sankey.link(); var nodes={nodes} var links={links} sankey .nodes(nodes) .links(links) .layout(32); var link = svg.append("g").selectAll(".link") .data(links) .enter().append("path") .attr("class", "link") .attr("d", path) .style("stroke-width", function(d) {{ return Math.max(1, d.dy); }}) .sort(function(a, b) {{ return b.dy - a.dy; }}); link.append("title") .text(function(d) {{ return d.source.name + " → " + d.target.name + "" + format(d.value); }}); var node = svg.append("g").selectAll(".node") .data(nodes) .enter().append("g") .attr("class", "node") .attr("transform", function(d) {{ return "translate(" + d.x + "," + d.y + ")"; }}) .call( d3.behavior.drag() .origin(function(d) {{ return d; }}) .on("dragstart", function() {{ this.parentNode.appendChild(this); }}) .on("drag", dragmove) ); node.append("rect") .attr("height", function(d) {{ return d.dy; }}) .attr("width", sankey.nodeWidth() ) .style("fill", function(d) {{ return d.color = color(d.name.replace(/ .*/, "")); }}) .style("stroke", function(d) {{ return d3.rgb(d.color).darker(2); }}) .append("title") .text(function(d) {{ return d.name + "" + format(d.value); }}); node.append("text") .attr("x", -6) .attr("y", function(d) {{ return d.dy / 2; }}) .attr("dy", ".35em") .attr("text-anchor", "end") .attr("transform", null) .text(function(d) {{ return d.name; }}) .filter(function(d) {{ return d.x < width / 2; }}) .attr("x", 6 + sankey.nodeWidth()) .attr("text-anchor", "start"); function dragmove(d){{ d3.select(this) .attr("transform", "translate(" + d.x + "," + (d.y = Math.max(0, Math.min(height - d.dy, d3.event.y))) + ")"); sankey.relayout(); link.attr("d", path); }} </script> '''.format(title=title, nolabel=nolabel, d3js_str=vjs.ViewJs.d3jsMin(), nodes=nodes, links=links, width=w, height=h) html.write(outTemplate) if not oFile == None: html.close()
def run(self, **kw_args): os.environ["KG_VerboseLevel"] = "2" if "msg" in kw_args: if kw_args["msg"] == "on": os.environ['KG_ScpVerboseLevel'] = "3" temp = Mtemp() xxedge = temp.file() xxnode = temp.file() xxnam2num = temp.file() xxnum2nam = temp.file() xxebase = temp.file() xxbody = temp.file() e1 = None if self.ew: e1 <<= nm.mcut(f="%s:__node1,%s:__node2,%s:__weight" % (self.ef1, self.ef2, self.ew), i=self.eFile) else: e1 <<= nm.mcut(f="%s:__node1,%s:__node2" % (self.ef1, self.ef2), i=self.eFile) e1 <<= nm.muniq(k="__node1,__node2") e2 = nm.mfldname(i=e1, f="__node2:__node1,__node1:__node2") fe = None fe <<= nm.muniq(k="__node1,__node2", i=[e1, e2], o=xxedge) fe.run() # cleaning the node data (remove duplicate nodes) fn = None if self.nFile: if self.nw: fn <<= nm.mcut(f="%s:__node,%s" % (self.nf, self.nw), i=self.nFile) else: fn <<= nm.mcut(f="%s:__node" % (self.nf), i=self.nFile) fn <<= nm.muniq(k="__node", o=xxnode) else: xxen1 = nm.mcut(f="__node1:__node", i=xxedge) xxen2 = nm.mcut(f="__node2:__node", i=xxedge) fn <<= nm.muniq(k="__node", o=xxnode, i=[xxen1, xxen2]) fn.run() # 節点名<=>節点番号変換表の作成 fmap = None fmap <<= nm.mcut(f="__node", i=xxnode) fmap <<= nm.mnumber(a="__num", S=1, q=True, o=xxnam2num) fmap <<= nm.msortf(f="__num", o=xxnum2nam) fmap.run() # 節点ファイルが指定された場合は枝ファイルとの整合性チェック if self.nFile: ncheck = nm.mcut(f="__node1:__node", i=xxedge) ncheck <<= nm.mcommon(k="__node", m=xxnam2num, r=True) nmatch = ncheck.run() if len(nmatch) > 0: raise Exception( "#ERROR# the node named '%s' in the edge file doesn't exist in the node file." % (nmatch[0][0])) # metisのグラフファイルフォーマット # 先頭行n m [fmt] [ncon] # n: 節点数、m:枝数、ncon: 節点weightの数 # 1xx: 節点サイズ有り (not used, meaning always "0") # x1x: 節点weight有り # xx1: 枝がweightを有り # s w_1 w_2 ... w_ncon v_1 e_1 v_2 e_2 ... v_k e_k # s: 節点サイズ (節点サイズは利用不可) # w_x: 節点weight # v_x: 接続のある節点番号(行番号) # e_x: 枝weight # -------------------- # generate edge data using the integer numbered nodes #fnnum = None fnnum = nm.mcut(f="__num:__node_n1", i=xxnam2num) # {xxnnum} fenum = None fenum <<= nm.mjoin(k="__node1", K="__node", f="__num:__node_n1", m=xxnam2num, i=xxedge) fenum <<= nm.mjoin(k="__node2", K="__node", f="__num:__node_n2", m=xxnam2num) fenum <<= nm.msortf(f="__node_n1") #{xxenum} febase = None febase <<= nm.mnjoin(k="__node_n1", m=fenum, i=fnnum, n=True) febase <<= nm.msortf(f="__node_n1%n,__node_n2%n", o=xxebase) #{xxebase}" febase.run() fbody = None if not self.ew: fbody <<= nm.mcut(f="__node_n1,__node_n2", i=xxebase) fbody <<= nm.mtra(k="__node_n1", f="__node_n2", q=True) fbody <<= nm.mcut(f="__node_n2", nfno=True, o=xxbody) # if ew= is specified, merge the weight data into the edge data. else: febody = None febody <<= nm.mcut(f="__node_n1,__node_n2:__v", i=xxebase) febody <<= nm.mnumber(S=0, I=2, a="__seq", q=True) fwbody = None fwbody <<= nm.mcut(f="__node_n1,__weight:__v", i=xxebase) fwbody <<= nm.mnumber(S=1, I=2, a="__seq", q=True) fbody <<= nm.msortf(f="__seq%n", i=[febody, fwbody]) fbody <<= nm.mtra(k="__node_n1", f="__v", q=True) fbody <<= nm.mcut(f="__v", nfno=True, o=xxbody) fbody.run() # xxbody # 2 7 3 8 5 9 # 1 7 3 10 5 11 7 12 # 1 8 2 10 4 13 7 14 # -------------------- # generate node data using integer number if self.nFile and self.nw: # xxnode # __node,v1,v2 # a,1,1 # b,1,1 # c,1,1 xxnbody = temp.file() xxnbody1 = temp.file() fnbody = None fnbody <<= nm.mjoin(k="__node", f="__num", i=xxnode, m=xxnam2num) fnbody <<= nm.msortf(f="__num%n") fnbody <<= nm.mcut(f=self.nw, nfno=True) fnbody <<= nm.cmd("tr ',' ' ' ") # tricky!! fnbody <<= nm.mwrite(o=xxnbody) fnbody.run() # xxnbody # 1 1 # 1 1 # 1 1 # paste the node weight with edge body fnbody1 = None fnbody1 <<= nm.mpaste(nfn=True, m=xxbody, i=xxnbody) fnbody1 <<= nm.cmd("tr ',' ' ' ") fnbody1 <<= nm.mwrite(o=xxnbody1) fnbody1.run() os.system("mv %s %s" % (xxnbody1, xxbody)) # xxbody # 1 1 2 7 3 8 5 9 # 1 1 1 7 3 10 5 11 7 12 # 1 1 1 8 2 10 4 13 7 14 eSize = mrecount(i=xxedge) eSize /= 2 nSize = mrecount(i=xxnode) nwFlag = 1 if self.nw else 0 ewFlag = 1 if self.ew else 0 fmt = "0%d%d" % (nwFlag, ewFlag) xxhead = temp.file() xxgraph = temp.file() os.system("echo '%d %d %s %d' > %s" % (nSize, eSize, fmt, self.ncon, xxhead)) os.system("cat %s %s > %s" % (xxhead, xxbody, xxgraph)) if self.mFile: nm.mfldname(f="__num:num,__node:node", i=xxnum2nam, o=self.mFile).run() if self.dFile: os.system("cp %s %s" % (xxgraph, self.dFile)) if not self.noexe: if self.verbose: os.system( "gpmetis -seed=%d -ptype=%s -ncuts=%d -ufactor=%d %s %d" % (self.seed, self.ptype, self.ncuts, self.ufactor, xxgraph, self.kway)) else: os.system( "gpmetis -seed=%d -ptype=%s -ncuts=%d -ufactor=%d %s %d > /dev/null" % (self.seed, self.ptype, self.ncuts, self.ufactor, xxgraph, self.kway)) import glob if len(glob.glob(xxgraph + ".part.*")) == 0: raise Exception( "#ERROR# command `gpmetis' didn't output any results") # 節点名を数字から元に戻す # #{xxgraph}.part.#{kway} # 1 # 0 # 1 fo = None fo <<= nm.mcut(f="0:cluster", nfni=True, i=xxgraph + ".part." + str(self.kway)) fo <<= nm.mnumber(S=1, a="__num", q=True) fo <<= nm.mjoin(k="__num", f="__node", m=xxnum2nam) fo <<= nm.msortf(f="__node,cluster") if self.nf: fo <<= nm.mcut(f="__node:%s,cluster" % (self.nf), o=self.oFile) else: fo <<= nm.mcut(f="__node:node,cluster", o=self.oFile) fo.run() nu.mmsg.endLog(self.__cmdline())
def run(self): tempW = mtemp.Mtemp() xxtra = tempW.file() xxmap1 = tempW.file() xxmap2 = tempW.file() lcmout = tempW.file() xxt0 = tempW.file() xxp0 = tempW.file() xx3t = tempW.file() xx4t = tempW.file() self.pair2tra(self.ei, self.ef1, self.ef2, xxtra, xxmap1, xxmap2) runPara = {} runPara["type"] = "CIf" runPara["sup"] = 1 runPara["o"] = lcmout runPara["i"] = xxtra if self.minSize2: runPara["l"] = self.minSize2 if self.maxSize2: runPara["u"] = self.maxSize2 extTake.lcm(runPara) extTake.lcmtrans(lcmout, "p", xxt0) f = None f <<= nm.mdelnull(f="pattern", i=xxt0) f <<= nm.mvreplace(vf="pattern", m=xxmap2, K="num2", f="node2") f <<= nm.mcut(f="pid,pattern,size:size2") f <<= nm.mvsort(vf="pattern") f <<= nm.msortf(f="pid") if self.byedge: f_e0 = nm.mtra(f="pattern", i=f, r=True) extTake.lcmtrans(lcmout, "t", xx3t) f_e1 = None f_e1 <<= nm.mjoin(k="__tid", m=xxmap1, f="node1", K="num1", i=xx3t) f_e1 <<= nm.msortf(f="pid") ## xx2 f_e2 = None f_e2 <<= nm.mcount(k="pid", a="size1", i=f_e1) f_e2 <<= nm.mselnum(f="size1", c="[{},{}]".format(self.minSize1, self.maxSize1)) f_e3 = None f_e3 <<= nm.mjoin(k="pid", m=f_e2, f="size1", i=f_e1) f_e3 <<= nm.mnjoin(k="pid", m=f_e0, f="pattern,size2") f_e3 <<= nm.mcut(f="pid:id,node1:{},pattern:{},size1,size2".format( self.ef1, self.ef2), o=self.oFile) f_e3.run() else: extTake.lcmtrans(lcmout, "t", xx4t) f_e4 = None f_e4 <<= nm.mjoin(k="__tid", m=xxmap1, i=xx4t, f="node1", K="num1") f_e4 <<= nm.mtra(k="pid", f="node1") f_e4 <<= nm.mvcount(vf="node1:size1") f_e4 <<= nm.mjoin(k="pid", m=f, f="pattern,size2") f_e4 <<= nm.mselnum(f="size1", c="[{},{}]".format(self.minSize1, self.maxSize1)) f_e4 <<= nm.mvsort(vf="node1,pattern") f_e4 <<= nm.msortf(f="node1,pattern") f_e4 <<= nm.mcut(f="node1:{},pattern:{},size1,size2".format( self.ef1, self.ef2), o=self.oFile) f_e4.run()