def toNum(): for size in ["all"]: iFile = "%s/online_all.csv" % datPath oFile1 = "%s/onlineT_all.csv" % datPath # data for Take.core oFile2 = "%s/onlineO_all.basket" % datPath # data for Orange oFile3 = "%s/onlineM_all.csv" % datPath # data for Take f = None f <<= nm.mcut(f="InvoiceNo,StockCode", i=iFile) f <<= nm.muniq(k="InvoiceNo,StockCode") f <<= nm.mfldname(q=True, o=oFile3) f.run(msg=debug) st = None st <<= nm.mcut(f="StockCode", i=iFile) st <<= nm.muniq(k="StockCode") st <<= nm.mnumber(s="StockCode", a="num") f = None f <<= nm.mjoin(k="StockCode", m=st, f="num", i=iFile) f <<= nm.mcut(f="InvoiceNo,num:StockCode") f <<= nm.mtra(k="InvoiceNo", f="StockCode") f <<= nm.mcut(f="StockCode", nfno=True, o=oFile1) f.run(msg=debug) os.system("tr ' ' ',' <%s >%s" % (oFile1, oFile2))
def __mkMap(key, nf, ni, ef1, ef2, ei, oFile): # leaf nodeの構築 infL = [ nm.mcommon(k=ef1, K=key, m=ei, r=True, i=ei).mcut(f="%s:nam" % (ef1)), nm.mcommon(k=ef2, K=key, m=ei, r=True, i=ei).mcut(f="%s:nam" % (ef2)) ] if ni: infL.append( nm.mcommon(k=nf, K=key, m=ei, r=True, i=ni).mcut(f="%s:nam" % (nf))) xleaf = nm.muniq(i=infL, k="nam") xleaf <<= nm.msetstr(v=1, a="leaf") if ni: inp = [ nm.mcut(f="%s:nam" % (nf), i=ni), nm.mcut(f="%s:nam" % (key), i=ni) ] else: inp = [ nm.mcut(f="%s:nam" % (ef1), i=ei), nm.mcut(f="%s:nam" % (ef2), i=ei), nm.mcut(f="%s:nam" % (key), i=ei) ] f = None f <<= nm.muniq(k="nam", i=inp) f <<= nm.mjoin(k="nam", m=xleaf, f="leaf", n=True) # nullは最初に来るはずなので、mcalでなくmnumberでもnullを0に採番できるはずだが念のために f <<= nm.mcal(c='if(isnull($s{nam}),0,line()+1)', a="num") f <<= nm.mnullto(f="nam", v="##NULL##", o=oFile) f.run()
def pair2tra(self, ei, ef1, ef2, traFile, mapFile1, mapFile2): f1 = None f1 <<= nm.mcut(f="{}:node1".format(self.ef1), i=self.ei) f1 <<= nm.mdelnull(f="node1") f1 <<= nm.muniq(k="node1") f1 <<= nm.mnumber(s="node1", a="num1", o=mapFile1) f2 = None f2 <<= nm.mcut(f="{}:node2".format(self.ef2), i=self.ei) f2 <<= nm.mdelnull(f="node2") f2 <<= nm.muniq(k="node2") f2 <<= nm.mnumber(s="node2", a="num2", o=mapFile2) f3 = None f3 <<= nm.mcut(f="{}:node1,{}:node2".format(self.ef1, self.ef2), i=self.ei) f3 <<= nm.mjoin(k="node1", m=f1, f="num1") f3 <<= nm.mjoin(k="node2", m=f2, f="num2") f3 <<= nm.mcut(f="num1,num2") f3 <<= nm.msortf(f="num1,num2%n") f3 <<= nm.mtra(k="num1", s="num2%n", f="num2") f3 <<= nm.msortf(f="num1%n") f3 <<= nm.mcut(f="num2", nfno=True, o=traFile) f3.run() os.system("cat " + traFile)
def edge2mtx(self, ei, itra, map1, map2): p1 = nm.mcut(f=self.ef1, i=ei) p1 <<= nm.muniq(k=self.ef1) p1 <<= nm.mdelnull(f=self.ef1) p1 <<= nm.mnumber(q=True, a="num1", S=1, o=map1) p2 = nm.mcut(f=self.ef2, i=ei) p2 <<= nm.muniq(k=self.ef2) p2 <<= nm.mdelnull(f=self.ef2) p2 <<= nm.mnumber(q=True, a="num2", S=1, o=map2) runp = None runp <<= nm.mcut(f=[self.ef1, self.ef2], i=ei) runp <<= nm.mjoin(k=self.ef1, m=p1, f="num1") runp <<= nm.mjoin(k=self.ef2, m=p2, f="num2") runp <<= nm.mcut(f="num1,num2") runp <<= nm.mtra(k="num1", f="num2") runp <<= nm.msortf(f="num1%n") runp <<= nm.mcut(f="num2", nfno=True) runp <<= nm.cmd("tr ',' ' '") runp <<= nm.mwrite(o=itra) #runp <<= nm.mcut(f="num2",nfno=True,o=wff1) runp.run()
def __convertToNumeric(self): wf1 = self.__tempW.file() wf2 = self.__tempW.file() wf3 = self.__tempW.file() self.mFile = self.__temp.file() nm.mcut(f="e1:node", i=self.__wfE, o=wf1).run() nm.mcut(f="e2:node", i=self.__wfE, o=wf2).run() mcmd = None if self.__wfN is not None: nm.mcut(f="n:node", i=self.__wfN, o=wf3).run() mcmd <<= nm.mcat(i=wf1 + "," + wf2 + "," + wf3, f="node") else: mcmd <<= nm.mcat(i=wf1 + "," + wf2, f="node") mcmd <<= nm.muniq(k="node") mcmd <<= nm.mnumber(q=True, a="id", o=self.mFile) mcmd.run() #エッジファイル変換・保存 self.eFile = self.__temp.file() mcmd = nm.mjoin(i=self.__wfE, m=self.mFile, f="id:id1", k="e1", K="node") mcmd <<= nm.mjoin(m=self.mFile, f="id:id2", k="e2", K="node") mcmd <<= nm.muniq(k="id1,id2") mcmd <<= nm.msortf(f="id1%n,id2%n") mcmd <<= nm.mcut(f="id1:e1,id2:e2,no:row_index", o=self.eFile) mcmd.run() #take内部用エッジファイル保存 self.eFileT = self.__temp.file() mcmd = nm.mcal(i=self.eFile, c="cat(\" \",$s{e1},$s{e2})", a="edge") mcmd <<= nm.mcut(nfno=True, f="edge", o=self.eFileT) mcmd.run() #ノードファイル変換・保存 self.nFile = self.__temp.file() if self.__wfN is not None: mcmd = nm.mjoin(i=self.__wfN, m=self.mFile, f="id", k="n", K="node") mcmd <<= nm.muniq(k="id") mcmd <<= nm.msortf(f="id%n") mcmd <<= nm.mcut(f="id:n,no:row_index", o=self.nFile) mcmd.run() else: #エッジよりノード生成 wf4 = self.__tempW.file() wf5 = self.__tempW.file() nm.mcut(i=self.eFile, f="e1:n", o=wf4).run() nm.mcut(i=self.eFile, f="e2:n", o=wf5).run() mcmd = nm.mcat(i="%s,%s" % (wf4, wf5)) mcmd <<= nm.muniq(k="n") mcmd <<= nm.msortf(f="n%n") mcmd <<= nm.mnumber(q=True, a="row_index", o=self.nFile) mcmd.run() #ワークファイル削除 self.__tempW.rm()
def readCSV(iParams): iFile = iParams["iFile"] sidF = iParams["sid"] eidF = iParams["time"] itemF = iParams["item"] temp = Mtemp() xxdatPath = temp.file() mkDir(xxdatPath) # classファイルの処理 if "cFile" in iParams: cFile = iParams["cFile"] csidF = iParams["csid"] classF = iParams["cNames"] f = None f <<= nm.mcut(f="%s:sid,%s:eid,%s:item" % (sidF, eidF, itemF), i=iFile) f <<= nm.mdelnull(f="sid,eid,item") f <<= nm.muniq(k="sid,eid,item") f <<= nm.mjoin(k="sid", K=csidF, m=cFile, f="%s:class" % (classF)) f <<= nm.msep(s="sid,eid%n,item", d="%s/${class}" % (xxdatPath), p=True) f.run() classNames = glob.glob("%s/*" % (xxdatPath)) classNames = [os.path.basename(path) for path in classNames] else: f = None f <<= nm.mcut(f="%s:sid,%s:eid,%s:item" % (sidF, eidF, itemF), i=iFile) f <<= nm.mdelnull(f="sid,eid,item") f <<= nm.muniq(k="sid,eid,item") f <<= nm.msortf(f="sid,eid%n,item", o="%s/single" % (xxdatPath)) f.run() classNames = ["single"] datas = {} for name in classNames: dataFile = "%s/%s" % (xxdatPath, name) datas[name] = _readCSV_sub(dataFile) return datas
def mnest2tree(ei, ef, k, ni=None, nf=None, ev=None, no=None, eo=None): # paracheck追加 efs = ef.split(",") ef1 = efs[0] ef2 = efs[1] f = nm.mcut(f="%s:#orgKey,%s:#orgEf1,%s:#orgEf2" % (k, ef1, ef2), i=ei) temp = mtemp.Mtemp() of = temp.file() with _nu.mcsvout(o=of, f="#orgKey,#orgEf1,#orgEf2,#ef1,#ef2") as oCSV: for flds in f: orgKey = flds[0] orgEf1 = flds[1] orgEf2 = flds[2] oCSV.write([orgKey, orgEf1, orgEf2, orgKey, orgEf1]) oCSV.write([orgKey, orgEf1, orgEf2, orgKey, orgEf2]) f = None f <<= nm.mjoin(k="#orgKey,#orgEf1,#orgEf2", K=[k, ef1, ef2], m=ei, i=of) # 全項目join if ev: f <<= nm.mavg(k="#ef1,#ef2", f=ev) else: f <<= nm.muniq(k="#ef1,#ef2") f <<= nm.mcut(r=True, f="#orgKey,#orgEf1,#orgEf2") f <<= nm.mfldname(f="#ef1:%s,#ef2:%s" % (ef1, ef2), o=eo) f.run() if ni: head = nu.mheader(i=ni) fldnames = [s for s in head if s != nf] commas = ',' * (len(fldnames) - 1) f0 = None f0 <<= nm.mcut(f="%s:%s" % (ef1, nf), i=eo) f0 <<= nm.muniq(k=nf) f0 <<= nm.mcommon(k=nf, m=ni, r=True) f0 <<= nm.msetstr(v=commas, a=fldnames) f = nm.mcut(f=k, r=True, i=[ni, f0]) f <<= nm.msetstr(v="", a=k, o=no) f.run()
def tra2tbl(iFile, oFile, tidFld, itemFld, null=0, dummy=True, aggFld=None, aggStat=None, klassFld=None): klass = None if klassFld is not None and klassFld != "": klass <<= nm.mcut(f=tidFld + "," + klassFld, i=iFile) klass <<= nm.muniq(k=tidFld) f = None # aggFldが指定されていればセル項目を集計 if aggFld is not None and aggFld != "": f <<= nm.mcut(f=tidFld + "," + itemFld + "," + aggFld, i=iFile) f <<= nm.mstats(k=tidFld, f="%s:_cell" % (aggFld), c=aggStat) # aggFldが指定されていなければカウント else: f <<= nm.mcut(f=tidFld + "," + itemFld, i=iFile) # アイテムが出現したかどうか if dummy: f <<= nm.muniq(k=tidFld + "," + itemFld) f <<= nm.msetstr(v=1, a="_cell") # アイテムが何件出現したかどうか else: f <<= nm.mcount(k=tidFld + "," + itemFld, a="_cell") # 横展開 f <<= nm.m2cross(k=tidFld, s=itemFld, f="_cell") # クラス項目が指定されていれば結合する if klassFld is not None and klassFld != "": f <<= nm.mjoin(k=tidFld, m=klass, f=klassFld) # null値を一斉に置換する f <<= nm.mnullto(f="*", v=null, o=oFile) f.run(msg="on")
def calGsize(self, file): edgesize = nu.mrecount(i=file, nfni=True) f = None f <<= nm.mcut(i=file, f="0:tra", nfni=True) f <<= nm.mtra(f="tra") f <<= nm.muniq(k="tra") f <<= nm.mcount(a="cnt") f <<= nm.mcut(f="cnt") f <<= nm.writelist(dtype="cnt:int") nodesize = f.run()[0][0] return nodesize, edgesize
def g2pair(self,ni,nf,ei,ef1,ef2,ew,numFile,mapFile,weightFile): #MCMD::msgLog("converting graph files into a pair of numbered nodes ...") #wf=MCMD::Mtemp.new #wf1=wf.file #wf2=wf.file #wf3=wf.file allinObj =[] wf1 = nm.mcut(f="%s:node"%(ef1),i=ei ).msetstr(v=0,a="flag") wf2 = nm.mcut(f="%s:node"%(ef2),i=ei ).msetstr(v=0,a="flag") f = None if nf: f <<= nm.mcut(i=[wf1,wf2,nm.mcut(f=nf+":node",i=ni).msetstr(v=1,a="flag")],f="node,flag") f <<= nm.mbest(k="node" , s="flag" , fr=0 ,size=1) else: f <<= nm.mcut(i=[wf1,wf2],f="node,flag") f <<= nm.muniq(k="node") f <<= nm.mnumber(s="flag,node",a="num",S=0,o=mapFile) f.run() f = None f <<= nm.mcut(f=[ef1,ef2],i=ei) f <<= nm.mjoin(k=ef1 , K="node",m=mapFile , f="num:num1") f <<= nm.mjoin(k=ef2 , K="node",m=mapFile , f="num:num2") f <<= nm.mcut(f="num1,num2") f <<= nm.mfsort(f="num1,num2") f <<= nm.msortf(f="num1%n,num2%n",nfno=True) f <<= nm.cmd("tr ',' ' ' ") f <<= nm.mwrite(o=numFile) f.run() if ew : nm.mcut(f=ew,i=ei,o=weightFile).run() else: ew="weight" nm.msetstr(v=1 , a=ew ,i=ei).mcut(f=ew,o=weightFile).run() nodeSize=mrecount(i=mapFile) return nodeSize
def __init__(self,iFile,itemFN,taxoFN): # アイテムの項目名(=>String) self.itemFN = None # 分類の項目名(=>String) self.taxoFN = None # アイテムの種類数(=>Fixnum) self.itemSize = None # 分類の種類数(=>Fixnum) self.taxoSize = None # taxonomyデータファイル名(=>String) self.file = None self.temp = mtemp.Mtemp() self.iFile = iFile self.iPath = os.path.abspath(self.iFile) self.itemFN = itemFN self.taxoFN = taxoFN # item順に並べ替えてpathに書き出す self.file = self.temp.file() para_it = self.itemFN +"," + self.taxoFN nm.mcut(f=para_it,i=self.iFile).muniq(k=para_it,o=self.file).run(msg="on") f = nm.mcut(f=self.itemFN,i=self.iFile) f <<= nm.mtrafld(f=self.itemFN,a="__fld",valOnly=True) f <<= nm.mtra(f="__fld",r=True) f <<= nm.muniq(k="__fld") f <<= nm.mcount(a="size") f <<= nm.mcut(f="size") xx1 = f.run() self.itemSize = int(xx1[0][0]) xx2 = nm.mcut(f=self.taxoFN+":item",i=self.file).muniq(k="item").mcount(a="size").mcut(f="size").run() self.taxoSize = int(xx2[0][0])
def search_h(self, b): field = self.field_w.getValue() value = self.value_w.value _head = self.head_w.value _tail = self.tail_w.value reverse = self.reverse_w.value self.parent.msg_w.value = "\"%s\" 項目の値 \"%s\" を検索中..." % (field, value) if _head and _tail: sub = False head = False tail = False elif _head: sub = False head = True tail = False elif _tail: sub = False head = False tail = True else: sub = True head = False tail = False #print(field,sub,head,tail,value) iFile = self.iName_w.value f = None f <<= nm.mcut(f=field, i=iFile) if value != "": f <<= nm.mselstr(f=field, sub=sub, head=head, tail=tail, r=reverse, v=value) f <<= nm.muniq(k=field) rsl = f.run() self.result_w.options = [v[0] for v in rsl] self.parent.msg_w.value = "\"%s\" 項目の値 \"%s\" を検索中...完了" % (field, value)
def g2pair(self, ni, nf, ei, ef1, ef2, ipair, mapFile): dlist = [ nm.mcut(f=ef1 + ":node", i=ei), nm.mcut(f=ef2 + ":node", i=ei) ] if (ni): dlist.append(nm.mcut(f=nf + ":node", i=ni)) fmap = None fmap <<= nm.mcut(i=dlist, f="node") fmap <<= nm.muniq(k="node") fmap <<= nm.mnumber(q=True, a="num", o=mapFile) f = nm.mcut(f=[ef1, ef2], i=ei) f <<= nm.mjoin(k=ef1, K="node", m=fmap, f="num:num1") f <<= nm.mjoin(k=ef2, K="node", m=fmap, f="num:num2") f <<= nm.mcut(f="num1,num2") f <<= nm.mfsort(f="num1,num2") f <<= nm.msortf(f="num1%n,num2%n", nfno=True, o=ipair) f.run()
def run(self, **kw_args): os.environ["KG_VerboseLevel"] = "2" if "msg" in kw_args: if kw_args["msg"] == "on": os.environ['KG_ScpVerboseLevel'] = "3" temp = Mtemp() xxedge = temp.file() xxnode = temp.file() xxnam2num = temp.file() xxnum2nam = temp.file() xxebase = temp.file() xxbody = temp.file() e1 = None if self.ew: e1 <<= nm.mcut(f="%s:__node1,%s:__node2,%s:__weight" % (self.ef1, self.ef2, self.ew), i=self.eFile) else: e1 <<= nm.mcut(f="%s:__node1,%s:__node2" % (self.ef1, self.ef2), i=self.eFile) e1 <<= nm.muniq(k="__node1,__node2") e2 = nm.mfldname(i=e1, f="__node2:__node1,__node1:__node2") fe = None fe <<= nm.muniq(k="__node1,__node2", i=[e1, e2], o=xxedge) fe.run() # cleaning the node data (remove duplicate nodes) fn = None if self.nFile: if self.nw: fn <<= nm.mcut(f="%s:__node,%s" % (self.nf, self.nw), i=self.nFile) else: fn <<= nm.mcut(f="%s:__node" % (self.nf), i=self.nFile) fn <<= nm.muniq(k="__node", o=xxnode) else: xxen1 = nm.mcut(f="__node1:__node", i=xxedge) xxen2 = nm.mcut(f="__node2:__node", i=xxedge) fn <<= nm.muniq(k="__node", o=xxnode, i=[xxen1, xxen2]) fn.run() # 節点名<=>節点番号変換表の作成 fmap = None fmap <<= nm.mcut(f="__node", i=xxnode) fmap <<= nm.mnumber(a="__num", S=1, q=True, o=xxnam2num) fmap <<= nm.msortf(f="__num", o=xxnum2nam) fmap.run() # 節点ファイルが指定された場合は枝ファイルとの整合性チェック if self.nFile: ncheck = nm.mcut(f="__node1:__node", i=xxedge) ncheck <<= nm.mcommon(k="__node", m=xxnam2num, r=True) nmatch = ncheck.run() if len(nmatch) > 0: raise Exception( "#ERROR# the node named '%s' in the edge file doesn't exist in the node file." % (nmatch[0][0])) # metisのグラフファイルフォーマット # 先頭行n m [fmt] [ncon] # n: 節点数、m:枝数、ncon: 節点weightの数 # 1xx: 節点サイズ有り (not used, meaning always "0") # x1x: 節点weight有り # xx1: 枝がweightを有り # s w_1 w_2 ... w_ncon v_1 e_1 v_2 e_2 ... v_k e_k # s: 節点サイズ (節点サイズは利用不可) # w_x: 節点weight # v_x: 接続のある節点番号(行番号) # e_x: 枝weight # -------------------- # generate edge data using the integer numbered nodes #fnnum = None fnnum = nm.mcut(f="__num:__node_n1", i=xxnam2num) # {xxnnum} fenum = None fenum <<= nm.mjoin(k="__node1", K="__node", f="__num:__node_n1", m=xxnam2num, i=xxedge) fenum <<= nm.mjoin(k="__node2", K="__node", f="__num:__node_n2", m=xxnam2num) fenum <<= nm.msortf(f="__node_n1") #{xxenum} febase = None febase <<= nm.mnjoin(k="__node_n1", m=fenum, i=fnnum, n=True) febase <<= nm.msortf(f="__node_n1%n,__node_n2%n", o=xxebase) #{xxebase}" febase.run() fbody = None if not self.ew: fbody <<= nm.mcut(f="__node_n1,__node_n2", i=xxebase) fbody <<= nm.mtra(k="__node_n1", f="__node_n2", q=True) fbody <<= nm.mcut(f="__node_n2", nfno=True, o=xxbody) # if ew= is specified, merge the weight data into the edge data. else: febody = None febody <<= nm.mcut(f="__node_n1,__node_n2:__v", i=xxebase) febody <<= nm.mnumber(S=0, I=2, a="__seq", q=True) fwbody = None fwbody <<= nm.mcut(f="__node_n1,__weight:__v", i=xxebase) fwbody <<= nm.mnumber(S=1, I=2, a="__seq", q=True) fbody <<= nm.msortf(f="__seq%n", i=[febody, fwbody]) fbody <<= nm.mtra(k="__node_n1", f="__v", q=True) fbody <<= nm.mcut(f="__v", nfno=True, o=xxbody) fbody.run() # xxbody # 2 7 3 8 5 9 # 1 7 3 10 5 11 7 12 # 1 8 2 10 4 13 7 14 # -------------------- # generate node data using integer number if self.nFile and self.nw: # xxnode # __node,v1,v2 # a,1,1 # b,1,1 # c,1,1 xxnbody = temp.file() xxnbody1 = temp.file() fnbody = None fnbody <<= nm.mjoin(k="__node", f="__num", i=xxnode, m=xxnam2num) fnbody <<= nm.msortf(f="__num%n") fnbody <<= nm.mcut(f=self.nw, nfno=True) fnbody <<= nm.cmd("tr ',' ' ' ") # tricky!! fnbody <<= nm.mwrite(o=xxnbody) fnbody.run() # xxnbody # 1 1 # 1 1 # 1 1 # paste the node weight with edge body fnbody1 = None fnbody1 <<= nm.mpaste(nfn=True, m=xxbody, i=xxnbody) fnbody1 <<= nm.cmd("tr ',' ' ' ") fnbody1 <<= nm.mwrite(o=xxnbody1) fnbody1.run() os.system("mv %s %s" % (xxnbody1, xxbody)) # xxbody # 1 1 2 7 3 8 5 9 # 1 1 1 7 3 10 5 11 7 12 # 1 1 1 8 2 10 4 13 7 14 eSize = mrecount(i=xxedge) eSize /= 2 nSize = mrecount(i=xxnode) nwFlag = 1 if self.nw else 0 ewFlag = 1 if self.ew else 0 fmt = "0%d%d" % (nwFlag, ewFlag) xxhead = temp.file() xxgraph = temp.file() os.system("echo '%d %d %s %d' > %s" % (nSize, eSize, fmt, self.ncon, xxhead)) os.system("cat %s %s > %s" % (xxhead, xxbody, xxgraph)) if self.mFile: nm.mfldname(f="__num:num,__node:node", i=xxnum2nam, o=self.mFile).run() if self.dFile: os.system("cp %s %s" % (xxgraph, self.dFile)) if not self.noexe: if self.verbose: os.system( "gpmetis -seed=%d -ptype=%s -ncuts=%d -ufactor=%d %s %d" % (self.seed, self.ptype, self.ncuts, self.ufactor, xxgraph, self.kway)) else: os.system( "gpmetis -seed=%d -ptype=%s -ncuts=%d -ufactor=%d %s %d > /dev/null" % (self.seed, self.ptype, self.ncuts, self.ufactor, xxgraph, self.kway)) import glob if len(glob.glob(xxgraph + ".part.*")) == 0: raise Exception( "#ERROR# command `gpmetis' didn't output any results") # 節点名を数字から元に戻す # #{xxgraph}.part.#{kway} # 1 # 0 # 1 fo = None fo <<= nm.mcut(f="0:cluster", nfni=True, i=xxgraph + ".part." + str(self.kway)) fo <<= nm.mnumber(S=1, a="__num", q=True) fo <<= nm.mjoin(k="__num", f="__node", m=xxnum2nam) fo <<= nm.msortf(f="__node,cluster") if self.nf: fo <<= nm.mcut(f="__node:%s,cluster" % (self.nf), o=self.oFile) else: fo <<= nm.mcut(f="__node:node,cluster", o=self.oFile) fo.run() nu.mmsg.endLog(self.__cmdline())
def run(self): wf = mtemp.Mtemp() xxpal = wf.file() xxa = wf.file() xxb = wf.file() xxc = wf.file() xxd = wf.file() xxout = wf.file() # ============ # n1,n2,sim # a,b,0.40 # a,c,0.31 # a,d,0.22 # b,c,0.20 # b,d,0.24 # b,e,0.14 # c,d,0.30 # d,e,0.09 xpal = None if self.directed: # 任意の枝a->bのaについて上位rankを選択 xpal <<= nm.mnumber(k=self.ef1, s=self.sim + "%nr", e="skip", S=1, a="##rank", i=self.ei) xpal <<= nm.mselnum(f="##rank", c="[," + str(self.rank) + "]") else: xxa = nm.mfsort(f=[self.ef1, self.ef2], i=self.ei) xxb = nm.mfsort(f=[self.ef2, self.ef1], i=self.ei) xpal <<= nm.muniq(k=[self.ef1, self.ef2], i=[xxa, xxb]) xpal <<= nm.mnumber(k=self.ef1, s=self.sim + "%nr", e="skip", S=1, a="##rank") xpal <<= nm.mselnum(f="##rank", c="[," + str(self.rank) + "]") # 両方向+片方向 xpal1 = None if self.dir == "x": xpal1 <<= nm.mcut(f=[self.ef1, self.ef2, self.sim], i=xpal) # 両方向 elif self.dir == "b": selpara = "$s{%s}==$s{##ef2}" % (self.ef1) # 得られた上位rankグラフからa->b->cを作成し、a==cであれば相思相愛ということ xpal1 <<= nm.mnjoin(k=self.ef2, K=self.ef1, m=xpal, f=self.ef2 + ":##ef2," + self.sim + ":sim2", i=xpal) xpal1 <<= nm.msel(c=selpara) xpal1 <<= nm.mcut(f=[self.ef1, self.ef2, self.sim]) else: selpara = "$s{%s}==$s{##ef2}" % (self.ef1) xxc = None xxc <<= nm.mnjoin(k=self.ef2, K=self.ef1, m=xpal, f=self.ef2 + ":##ef2," + self.sim + ":sim2", i=xpal) xxc <<= nm.msel(c=selpara) xxc <<= nm.mcut(f=[self.ef1, self.ef2]) xpal1 <<= nm.mcut(f=[self.ef1, self.ef2, self.sim], i=xpal) xpal1 <<= nm.mcommon(k=self.ef1 + "," + self.ef2, m=xxc, r=True) runpal = None kpara = "%s,%s" % (self.ef1, self.ef2) if self.udout: runpal <<= nm.mfsort(f=kpara, i=xpal1) runpal <<= nm.mavg(k=kpara, f=self.sim) runpal <<= nm.msortf(f=kpara, o=self.eo) else: runpal <<= nm.msortf(f=kpara, i=xpal1, o=self.eo) runpal.run() if self.ni and self.no: shutil.copyfile(self.ni, self.no)
def enumerate(self,eArgs): tf=nu.Mtemp() # 最小サポートと最小サポート件数 if "minCnt" in eArgs : self.minCnt = int(eArgs["minCnt"]) self.minSup = float(self.minCnt)/ float(self.db.size) else: self.minSup = float(eArgs["minSup"]) self.minCnt = int(self.minSup * float(self.db.size) + 0.99) # 最大サポートと最大サポート件数 self.maxCnt=None if "maxCnt" in eArgs or "maxSup" in eArgs: if "maxCnt" in eArgs: self.maxCnt = int(eArgs["maxCnt"]) self.maxSup = float(self.maxCnt)/float(self.db.size) else: self.maxSup = float(eArgs["maxSup"]) self.maxCnt = int(self.maxSup * float(self.db.size) + 0.99) #未使用 #@minProb = eArgs["minProb"].to_f # 事後確率 #@minGR = @minProb/(1-@minProb) # 増加率 #@minGR = eArgs["minGR"].to_f if eArgs["minGR"] # あるクラスをpos、他のクラスをnegにして、パターン列挙した結果ファイル名を格納する pFiles=[] tFiles=[] for cName,posSize in self.db.clsNameRecSize.items(): negSize=self.db.size-posSize # minGRの計算 if "minGR" in eArgs: self.minGR=eArgs["minGR"] else: minProb = eArgs["minProb"] if ( "minProb" in eArgs ) else 0.5 if "uniform" in eArgs and eArgs["uniform"]: self.minGR = (minProb/(1-minProb)) * (self.db.clsSize-1) # マニュアルの式(4) else: self.minGR = (minProb/(1-minProb)) * (float(negSize)/float(posSize)) # マニュアルの式(4) # 最小サポートと最小サポート件数 if "minCnt" in eArgs: self.minPos = eArgs["minCnt"] else: self.minPos = int(eArgs["minSup"] * float(posSize) + 0.99) # 最大サポートと最大サポート件数 if "maxCnt" in eArgs or "maxSup" in eArgs: if "maxCnt" in eArgs: self.maxCnt = int(eArgs["maxCnt"]) else: self.maxCnt = int(eArgs["maxSup"] * float(posSize) + 0.99) self.sigma[cName] = self.calSigma(self.minPos,self.minGR,posSize,negSize) # lcm_seqのパラメータ設定と実行 lcmout = tf.file() # lcm_seq出力ファイル # 頻出パターンがなかった場合、lcm出力ファイルが生成されないので # そのときのために空ファイルを生成しておいく。 with open(lcmout, "w") as efile: pass params = {} if self.msgoff: params["type"] ="CIA_" else: params["type"] ="CIA" if self.maxCnt: # windowサイズ上限 params["U"] = str(self.maxCnt) if "minLen" in eArgs: params["l"] = str(eArgs["minLen"]) if 'maxLen' in eArgs: params["u"] = str(eArgs["maxLen"]) if 'gap' in eArgs: params["g"] = str(eArgs["gap"]) if 'win' in eArgs: params["G"] = str(eArgs["win"]) params["w"] = self.weightFile[cName] params["i"] = self.file params["sup"] = str(self.sigma[cName]) params["o"] = lcmout # lcm_seq実行 #MCMD::msgLog("#{run}") if 'padding' in eArgs and eArgs["padding"]: # padding指定時は、0アイテムを出力しないlcm_seqを実行 extTake.lcmseq_zero(params) else: extTake.lcmseq(params) # パターンのサポートを計算しCSV出力する #MCMD::msgLog("output patterns to CSV file ...") pFiles.append(self.temp.file()) transle = self.temp.file() extTake.lcmtrans(lcmout,"e",transle) # pattern,countP,countN,size,pid f=None f <<= nm.mdelnull(f="pattern",i=transle) f <<= nm.mcal(c='round(${countN},1)',a="neg") f <<= nm.mcal(c='round(${countP}/%s,1)'%(self.posWeight[cName]),a="pos") f <<= nm.mdelnull(f="pattern") f <<= nm.msetstr(v=cName,a="class") f <<= nm.msetstr(v=posSize,a="posTotal") f <<= nm.msetstr(v=self.minGR,a="minGR") f <<= nm.mcut(f="class,pid,pattern,size,pos,neg,posTotal,minGR",o=pFiles[-1]) f.run() #s = MCMD::mrecount("i=#{pFiles.last}") # 列挙されたパターンの数 #MCMD::msgLog("the number of contrast patterns on class `#{cName}' enumerated is #{s}") if self.outtf : # トランザクション毎に出現するシーケンスを書き出す #MCMD::msgLog("output tid-patterns ...") tFiles.append(self.temp.file()) xxw= tf.file() f=None f <<= nm.mcut(f=self.db.idFN,i=self.db.file) f <<= nm.muniq(k=self.db.idFN) f <<= nm.mnumber(S=0,a="__tid",q=True) f <<= nm.msortf(f="__tid",o=xxw) f.run() nm.mcut(f=self.db.idFN,i=self.db.file).muniq(k=self.db.idFN).mnumber(S=0,a="__tid",q=True,o=xxw).run() translt = self.temp.file() extTake.lcmtrans(lcmout,"t",translt) nm.mjoin(k="__tid",m=xxw,f=self.db.idFN,i=translt).msetstr(v=cName,a="class").mcut(f=self.db.idFN+",class,pid",o=tFiles[-1]).run() # クラス別のパターンとtid-pidファイルを統合して最終出力 self.pFile = self.temp.file() self.tFile = self.temp.file() # パターンファイル併合 xxpCat = tf.file() f = nm.mcat(i=",".join(pFiles)) f <<= nm.msortf(f="class,pid") f <<= nm.mnumber(s="class,pid",S=0,a="ppid",o=xxpCat) f.run() # パターンファイル計算 items=self.db.items f="" f = nm.mcut(f="class,ppid:pid,pattern,size,pos,neg,posTotal,minGR",i=xxpCat) f <<= nm.msetstr(v=self.db.size,a="total") f <<= nm.mcal(c='${total}-${posTotal}',a="negTotal") # negのトータル件数 f <<= nm.mcal(c='${pos}/${posTotal}',a="support") # サポートの計算 f <<= nm.mcal(c='if(${neg}==0,1.797693135e+308,(${pos}/${posTotal})/(${neg}/${negTotal}))',a="growthRate") if "uniform" in eArgs and eArgs["uniform"] == True: f <<= nm.mcal(c='(${pos}/${posTotal})/(${pos}/${posTotal}+(%s-1)*${neg}/${negTotal})'%(self.db.clsSize),a="postProb") else: f <<= nm.mcal(c='${pos}/(${pos}+${neg})',a="postProb") f <<= nm.msel(c='${pos}>=%s&&${growthRate}>=${minGR}'%(self.minPos)) # minSupとminGRによる選択 f <<= nm.mvreplace(vf="pattern",m=items.file,K=items.idFN,f=items.itemFN) f <<= nm.mcut(f="class,pid,pattern,size,pos,neg,posTotal,negTotal,total,support,growthRate,postProb") f <<= nm.mvsort(vf="pattern") f <<= nm.msortf(f="class%nr,postProb%nr,pos%nr",o=self.pFile) f.run() if self.outtf : # 列挙されたパターンを含むtraのみ選択するためのマスタ xxp4=nm.mcut(f="class,pid",i=self.pFile) f = nm.mcat(i=",".join(tFiles)) f <<= nm.mjoin(k="class,pid",m=xxpCat,f="ppid") # 全クラス統一pid(ppid)結合 f <<= nm.mcommon(k="class,ppid",K="class,pid",m=xxp4) # 列挙されたパターンの選択 f <<= nm.mcut(f=self.db.idFN+",class,ppid:pid") f <<= nm.msortf(f=self.db.idFN+",class,pid",o=self.tFile) f.run() self.size = nu.mrecount(i=self.pFile)
tra <<= nm.mjoin(k="date", m=topix, f="i") tra <<= nm.mslide(k="id", s="date", f="date:date2,c:c2,i:i2") tra <<= nm.mcal(c="${c2}/${c}-${i2}/${i}", a="ret") tra <<= nm.mselnum(f="ret", c="[0.05,0.1]") tra <<= nm.mcut(f="id,date2:date,ret") # frequency of one item freq = None freq <<= nm.mcut(f="id", i=tra) freq <<= nm.mcount(k="id", a="freq") freq <<= nm.mselnum(f="freq", c="[5,]") # total number of transactions total = None total <<= nm.mcut(f="date", i=tra) total <<= nm.muniq(k="date") total <<= nm.mcount(a="total") # frequency of cooccurence of id, and calculate lift values itemCoFreq = None itemCoFreq <<= nm.mcut(f="date,id", i=tra) itemCoFreq <<= nm.mcommon(k="id", m=freq) itemCoFreq <<= nm.mcombi(k="date", n=2, f="id", a="id1,id2") itemCoFreq <<= nm.mcut(f="id1,id2") itemCoFreq <<= nm.mfsort(f="id1,id2") itemCoFreq <<= nm.mcount(k="id1,id2", a="coFreq") itemCoFreq <<= nm.mjoin(k="id1", m=freq, K="id", f="freq:freq1") itemCoFreq <<= nm.mjoin(k="id2", m=freq, K="id", f="freq:freq2") itemCoFreq <<= nm.mproduct(m=total, f="total") itemCoFreq <<= nm.mcal(c="(${coFreq}*${total})/(${freq1}*${freq2})", a="lift") itemCoFreq <<= nm.msel(c="${lift}>10 && ${coFreq}>6")
def mnetpie(ei, ni, ef, nf, o, nodeSizeFld=None, nodeTipsFld=None, nodeColorFld=None, edgeWidthFld=None, edgeColorFld=None, pieDataFld=None, pieTipsFld=None, picFld=None, undirect=False, offline=False): #ei:edge file #ef:egfile if type(ef) is str: ef = ef.split(',') if len(ef) != 2: raise Exception("ef= takes just two field names") if not ((pieDataFld == None and pieTipsFld == None) or (pieDataFld != None and pieTipsFld != None)): raise Exception( "pieDataFld= pieTipsFld= are necessary at the same time") if picFld != None and pieDataFld != None: raise Exception( "picFld= cannot be specified with pieDataFld= pieTipsFld=") if nodeColorFld != None: if picFld != None or pieDataFld != None or pieTipsFld != None: raise Exception( "nodeColorFld= cannot be specified with pieDataFld= pieTipsFld= picFld=" ) if pieDataFld != None and pieTipsFld != None: caseNo = 1 elif picFld != None: caseNo = 2 else: caseNo = 0 tempW = mtemp.Mtemp() xxnode = tempW.file() nodefld = [] nodedmy1 = [] nodedmy2 = [] nodefld.append("%s:node" % (nf)) if nodeSizeFld != None: nodefld.append("%s:nodesize" % (nodeSizeFld)) else: nodedmy1.append("nodesize") nodedmy2.append("50") if nodeTipsFld != None: nodefld.append("%s:nodeT" % (nodeTipFld)) else: nodedmy1.append("nodeT") nodedmy2.append("") if nodeColorFld != None: nodefld.append("%s:nodeClr" % (nodeColorFld)) else: nodedmy1.append("nodeClr") nodedmy2.append("skyblue") if caseNo == 1: nodefld.append("%s:pieD" % (pieDataFld)) nodefld.append("%s:pieT" % (pieTipsFld)) elif caseNo == 2: nodefld.append("%s:pic" % (picFld)) else: nodedmy1.append("pic") nodedmy2.append("") f1 = None f1 <<= nm.mcut(i=ni, f=nodefld) if len(nodedmy1) != 0: f1 <<= nm.msetstr(a=nodedmy1, v=nodedmy2) if caseNo == 1: f1 <<= nm.mshare(k="node", f="pieD:pieDS") f1 <<= nm.mnumber(k="node", a="nodeid", B=True) f2 = nm.muniq(k="pieT", i=f1) f2 <<= nm.mnumber(q=True, a="pieTno") f2 <<= nm.mjoin(k="pieT", f="pieTno", i=f1).iredirect("m") f2 <<= nm.msortf(f="nodeid%n,pieTno%n", o=xxnode) else: f2 = nm.mnumber(a="nodeid%n", q=True, i=f1, o=xxnode) f2.run() xxedge = tempW.file() # MAKE EDGE DATA edgefld = [] edgedmy1 = [] edgedmy2 = [] edgefld.append("%s:edgeS" % (ef[0])) edgefld.append("%s:edgeE" % (ef[1])) if edgeWidthFld != None: edgefld.append("%s:edgesize" % (edgeWidthFld)) else: edgedmy1.append("edgesize") edgedmy2.append("1") if edgeColorFld != None: edgefld.append("%s:edgecolor" % (edgeColorFld)) else: edgedmy1.append("edgecolor") edgedmy2.append("black") f3 = None f3 <<= nm.mcut(i=ei, f=edgefld) if len(edgedmy1) != 0: f3 <<= nm.msetstr(a=edgedmy1, v=edgedmy2) f3 <<= nm.mnumber(a="preNo", q=True) f3 <<= nm.mbest(k="edgeS,edgeE", s="preNo%nr") f3 <<= nm.mnumber(s="preNo%n", a="edgeID") f3 <<= nm.mjoin(k="edgeS", K="node", f="nodeid:edgeSid", m=xxnode) f3 <<= nm.mjoin(k="edgeE", K="node", f="nodeid:edgeEid", m=xxnode) #双方向チェック一応 f4 = None f4 <<= nm.mfsort(i=f3, f="edgeS,edgeE") f4 <<= nm.mcount(k="edgeS,edgeE", a="edgecnt") f4 <<= nm.mselnum(c="[2,]", f="edgecnt") f4 <<= nm.msetstr(a="biflg", v=1) f4 <<= nm.mjoin(k="edgeID", f="biflg", n=True, i=f3).iredirect("m") f4 <<= nm.msortf(f="edgeID%n", o=xxedge) f4.run() gdata = "{\"nodes\":[" if caseNo == 1: nodedatastk = [] nodedatas = "" for val, top, bot in nm.readcsv(xxnode).getline(k="nodeid", otype='dict', q=True): name = val["node"] r = val["nodesize"] title = val["nodeT"] if top: nodedatas = "{\"name\":\"%s\",\"title\":\"%s\",\"r\":%s,\"node\":[" % ( name, title, r) pieTno = val["pieTno"] pieT = val["pieT"] pieDS = val["pieDS"] nodedatas += "{\"group\":%s,\"color\":%s,\"value\":%s,\"title\":\"%s\"}" % ( pieTno, pieDS, pieDS, pieT) if bot: nodedatas += "]}" nodedatastk.append(nodedatas) nodedatas = "" else: nodedatas += "," gdata += ",".join(nodedatastk) else: nodedatastk = [] for val in nm.readcsv(xxnode).getline(otype='dict'): name = val["node"] r = val["nodesize"] title = val["nodeT"] pic = val["pic"] nclr = val["nodeClr"] nodedatas = "{\"name\":\"%s\",\"title\":\"%s\",\"pic\":\"%s\",\"color\":\"%s\",\"r\":%s}" % ( name, title, pic, nclr, r) nodedatastk.append(nodedatas) gdata += ",".join(nodedatastk) gdata += "],\"links\": [" edgedatastk = [] for val in nm.readcsv(xxedge).getline(otype='dict'): es = val["edgeSid"] et = val["edgeEid"] esize = val["edgesize"] ecolor = val["edgecolor"] edgedatas = "{\"source\":%s,\"target\":%s,\"length\":500,\"ewidth\":%s,\"color\":\"%s\"}" % ( es, et, esize, ecolor) edgedatastk.append(edgedatas) gdata += ','.join(edgedatastk) gdata += "]}" direct = ".attr('marker-end','url(#arrowhead)')" if undirect: direct = "" nodeTemplate = ''' node .append("circle") .attr("r",function(d){return d.r/4;}) .attr("fill", function(d){return d.color;}) .append("title") .text(function(d){return d.title;}) ''' nodemakeTemplate = ''' for(var i=0 ; i< graph.nodes.length;i++){ graph.nodes[i].id = i } ''' if pieDataFld != None: nodeTemplate = ''' node.selectAll("path") .data( function(d, i){ return pie(d.node); }) .enter() .append("svg:path") .attr("d", arc) .attr("fill", function(d, i) { return color(d.data.group); }) .append("title") .text(function(d){{return d.data.title;}}) node.append("circle") .attr("r",function(d){{return d.r/4;}}) .attr({ 'fill': 'white' }) .append("title") .text(function(d){{return d.title;}}); ''' nodemakeTemplate = ''' for(var i=0 ; i< graph.nodes.length;i++){ var r = graph.nodes[i].r for(var j=0 ; j< graph.nodes[i].node.length;j++){ graph.nodes[i].node[j]['r'] = r } graph.nodes[i].id = i } ''' elif picFld != None: nodeTemplate = ''' node .append("image") .attr("height",function(d){return d.r;}) .attr("width",function(d){return d.r;}) .attr("x",function(d){return -1 * d.r/2; }) .attr("y",function(d){return -1 * d.r/2; }) .attr("xlink:href",function(d){return d.pic; }) .append("title") .text(function(d){return d.title;}) ''' d3js_str = "<script type='text/javascript' src='http://d3js.org/d3.v3.min.js'></script>" if offline: d3js_str = "<script>%s<script>" % (vjs.ViewJs.d3jsMin()) outTemplate = ''' <!DOCTYPE html> <html lang="en"> <head> <meta charset="utf-8"> {d3js_str} <style></style> </head> <body> <script type="text/javascript"> var graph = {gdata} ; var width = 4000, height = 3000; var color = d3.scale.category10(); {nodemakeTemplate}; for(var i=0 ; i< graph.links.length;i++){{ graph.links[i].id = i }} var pie = d3.layout.pie() .sort(null) .value(function(d) {{ return d.value; }}); var arc = d3.svg.arc() .outerRadius( function(d){{ return d.data.r ; }}) .innerRadius( function(d){{ return d.data.r/2 ; }} ); var svg = d3.select("body").append("svg") .attr("width", width) .attr("height", height); d3.select("svg").append('defs').append('marker') .attr({{'id':'arrowhead', 'viewBox':'-0 -5 10 10', 'refX':30, 'refY':0, 'orient':'auto-start-reverse', 'markerWidth':5, 'markerHeight':5, 'xoverflow':'visible'}}) .append('path') .attr('d', 'M 0,-5 L 10 ,0 L 0,5') .attr('fill', '#999') .style('stroke','none'); var g = svg.append("g"); var node = g.selectAll(".node"); var link = g.selectAll(".link"); nodes = graph.nodes links = graph.links var force = d3.layout.force() .linkDistance(200) .linkStrength(3.5) .charge(-3500) .gravity(0.1) .friction(0.95) .size([width, height]) .on("tick", function() {{ link .attr("x1", function(d) {{ return d.source.x; }}) .attr("y1", function(d) {{ return d.source.y; }}) .attr("x2", function(d) {{ return d.target.x; }}) .attr("y2", function(d) {{ return d.target.y; }}); node .attr("x", function(d) {{ return d.x; }}) .attr("y", function(d) {{ return d.y; }}) .attr("transform", function(d) {{ return "translate(" + d.x + "," + d.y + ")"}}); }}); node = node.data(nodes, function( d ) {{ return d.id; }} ); link = link.data(links, function( d ) {{ return d.id; }} ); link .enter() .append("line") .attr("class", "link") .style("stroke", function( d ) {{ return d.color; }} ) .style("stroke-width", function( d ) {{ return d.ewidth; }}) {direct} node .enter() .append("g") .attr("class", "node") .style({{}}) .call(force.drag) .on("contextmenu", function(nd) {{ d3.event.preventDefault(); force.stop() nodes.splice( nd.index, 1 ); links = links.filter(function(nl) {{ return nl.source.index != nd.index && nl.target.index != nd.index; }}); node = node.data(nodes, function( d ) {{ return d.id; }} ); node.exit().remove(); link = link.data( links, function( d ) {{ return d.id; }} ); link.exit().remove(); force.nodes(nodes) .links(links) .start(); }}); {nodeTemplate} node .append("text") .attr("text-anchor", "middle") .style("stroke", "black") .text(function(d) {{ return d.name; }}); force.nodes(nodes) .links(links) .start(); </script> </body> </html> '''.format(d3js_str=d3js_str, gdata=gdata, nodemakeTemplate=nodemakeTemplate, direct=direct, nodeTemplate=nodeTemplate) html = sys.stdout if not o == None: html = open(o, "w") html.write(outTemplate) if not o == None: html.close()
def run(self): from datetime import datetime t = datetime.now() wf = nu.Mtemp() xxinp = wf.file() xxmap = wf.file() input = self.ei self.g2pair(self.ni, self.nf, self.ei, self.ef1, self.ef2, xxinp, xxmap) xxmace = wf.file() # maceの出力(tra形式) para = {} if self.msgoff: para["type"] = "Ce_" if self.all else "Me_" else: para["type"] = "Ce" if self.all else "Me" para["i"] = xxinp para["o"] = xxmace if self.minSize: para["l"] = self.minSize if self.maxSize: para["u"] = self.maxSize extTake.mace(para) #MCMD::msgLog("converting the numbered nodes into original name ...") id = nu.mrecount(i=xxmace, nfni=True) # xxpair = wf.file() # 上記traをpair形式に変換したデータ fpair = None fpair <<= nm.mcut(i=xxmace, nfni=True, f="0:num") fpair <<= nm.mnumber(q=True, a="id") fpair <<= nm.mvcount(vf="num:size") fpair <<= nm.mtra(r=True, f="num") # when ni= specified, it add the isolated single cliques. if self.ni: fpair_u = nm.mread(i=fpair) if self.all: fpair_u <<= nm.mselstr(f="size", v=1) fpair_u <<= nm.mcut(f="num") fpair_u <<= nm.muniq(k="num") # select all nodes which are not included in any cliques xxiso = None xxiso <<= nm.mcut(f="num", i=xxmap) xxiso <<= nm.mcommon(k="num", m=fpair_u, r=True) xxiso <<= nm.mnumber(S=id, a="id", q=True) xxiso <<= nm.msetstr(v=1, a="size") xxiso <<= nm.mcut(f="id,num,size") # mcut入れないとおかしくなるあとで直す #ddlist = [fpair.mcut(f="id,num,size"),xxiso] xxpair = nm.mcut(i=[fpair, xxiso], f="id,num,size") else: xxpair = fpair xxpair <<= nm.mjoin(m=xxmap, k="num", f="node") xxpair <<= nm.mcut(f="id,node,size") xxpair <<= nm.msortf(f="id,node", o=self.oFile) xxpair.run() procTime = datetime.now() - t # ログファイル出力 if self.logFile: kv = [["key", "value"]] for k, v in self.args.items(): kv.append([k, str(v)]) kv.append(["time", str(procTime)]) nm.writecsv(i=kv, o=self.logFile).run()
def __mkFlat(iFile, oFile): f = None f <<= nm.mcut(f="keyNum:node0", i=iFile) f <<= nm.muniq(k="node0", o=oFile) f.run() return 1
##### 上の2行は、以下に置き換え ##### import nysol.take as nt iFile = ("./DATA/online_all.csv") oPath = ("./OUTPUT/friends") os.system("mkdir -p %s" % oPath) # Make a similarity graph of StockCode: frequent 2-itemset enumeration # iFile # InvoiceNo,StockCode,Description,Quantity,InvoiceDate,UnitPrice,CustomerID,Country # 536365,85123A,WHITE HANGING HEART T-LIGHT HOLDER,6,2010/12/1 8:26,2.55,17850,United Kingdom # 536365,71053,WHITE METAL LANTERN,6,2010/12/1 8:26,3.39,17850,United Kingdom f = None f <<= nm.mcut(f="InvoiceNo,StockCode", i=iFile) f <<= nm.muniq(k="InvoiceNo,StockCode", o="%s/tra.csv" % oPath) f.run(msg="on") #args=Margs(["dummy","S=100","tid=InvoiceNo","item=StockCode","i=%s/tra.csv"%oPath,"O=%s"%oPath,"l=2","u=2"],"i=,x=,O=,tid=,item=,class=,taxo=,type=,s=,S=,sx=,Sx=,g=,p=,-uniform,l=,u=,top=,T=,-replaceTaxo") #mitemset(args).run() ##### 上の2行は、以下で動くようにする nt.mitemset(S=100, tid="InvoiceNo", item="StockCode", l=2, u=2, i="%s/tra.csv" % oPath, O=oPath).run() # patterns.csv # pid,size,count,total,support%0nr,lift,pattern
def __init__(self, iFile, idFN, timeFN, itemFN, padding, clsFN=None): self.file = None # トランザクションファイル名 self.idFN = None # トランザクションID項目名(String) self.timeFN = None # 時間項目名(String) self.itemFN = None # アイテム集合項目名(String) self.clsFN = None # クラス項目名(String) self.size = None # トランザクションサイズ(Num) self.items = None # Itemsクラス self.taxonomy = None # 階層分類クラス self.clsNameRecSize = None # クラス別件数 self.clsSize = None # クラス数 self.cFile = None # クラスファイル self.temp = mtemp.Mtemp() self.iFile = iFile # 入力ファイル self.iPath = os.path.abspath(self.iFile) # フルパス self.idFN = idFN # トランザクションID項目名 self.timeFN = timeFN # 時間項目名 self.itemFN = itemFN # アイテム項目名 self.file = self.temp.file() # 出力ファイル名 self.padding = padding # 時系列パディング # padding指定の場合は、asciiコードの先頭文字列を挿入 f = None if self.padding: f <<= nm.mcut(f=self.idFN + "," + self.timeFN + "," + self.itemFN, i=self.iFile) f <<= nm.msortf(f=self.idFN + "," + self.timeFN + "%n," + self.itemFN) f <<= nm.muniq(k=self.idFN + "," + self.timeFN + "," + self.itemFN) f <<= nm.mpadding(k=self.idFN, f=self.timeFN + "%n", v="!", o=self.file) else: f <<= nm.mcut(f=self.idFN + "," + self.timeFN + "," + self.itemFN, i=self.iFile) f <<= nm.msortf(f=self.idFN + "," + self.timeFN + "%n," + self.itemFN) f <<= nm.muniq(k=self.idFN + "," + self.timeFN + "," + self.itemFN, o=self.file) f.run() # レコード数の計算 #@recCnt = MCMD::mrecount("i=#{@file}") # トランザクション数の計算 xx1 = nm.mcut(f=self.idFN, i=self.file).muniq(k=self.idFN).mcount( a="__cnt").mcut(f='__cnt').run() self.size = int(xx1[0][0]) # トランザクションデータからアイテムオブジェクトを生成 if self.padding: #同じ? # paddingの場合は、特殊アイテム(!)を0番とし、出力を制御する。 self.items = items.Items(self.file, self.itemFN) else: self.items = items.Items(self.file, self.itemFN) if clsFN: self.clsFN = clsFN self.cFile = self.temp.file() fpara_c = "%s,%s" % (self.idFN, self.clsFN) nm.mcut(f=fpara_c, i=self.iFile).muniq(k=fpara_c, o=self.cFile).run() # 文字列としてのクラス別件数配列を数値配列に変換する self.clsSize = 0 self.clsNames = [] self.clsNameRecSize = {} for vv in nm.mcut(f=self.clsFN, i=self.cFile).mcount(k=self.clsFN, a='count'): self.clsNames.append(vv[0]) self.clsNameRecSize[vv[0]] = int(vv[1]) self.clsSize += 1
def msankey(i, o, v, f, title="", h=500, w=960, nl=False, T=None): # f= 2 fld if type(f) is str: ff = f.split(',') elif type(f) is list: ff = f else: raise TypeError("f= unsupport " + str(type(k))) if len(ff) < 2: raise TypeError("f= takes just two field names") if T != None: import re os.environ["KG_TmpPath"] = re.sub(r'/$', "", T) if h == None: h = 500 if w == None: w = 960 if title == None: title = "" tempW = mtemp.Mtemp() nodef = tempW.file() edgef = tempW.file() ef1 = ff[0] ef2 = ff[1] ev = v iFile = i oFile = o f0 = nm.mcut(i=iFile, f="%s:nodes" % (ef1)) f1 = nm.mcut(i=iFile, f="%s:nodes" % (ef2)) f2 = None f2 <<= nm.muniq(i=[f0, f1], k="nodes") f2 <<= nm.mnumber(s="nodes", a="num", o=nodef) f2.run() f3 = None f3 <<= nm.mcut(f="%s:nodes1,%s:nodes2,%s" % (ef1, ef2, ev), i=iFile) f3 <<= nm.mjoin(k="nodes1", K="nodes", m=nodef, f="num:num1") f3 <<= nm.mjoin(k="nodes2", K="nodes", m=nodef, f="num:num2") f3 <<= nm.mcut(f="num1,num2,%s" % (ev)) f3 <<= nm.msortf(f="num1%n,num2%n", o=edgef) f3.run() wk = [] nodeL = [] for flds in nm.readcsv(nodef).getline(otype='dict'): nodeL.append({"name": flds['nodes']}) nodes = json.JSONEncoder().encode(nodeL) linkL = [] for flds in nm.readcsv(edgef).getline(otype='dict', dtype={ "num1": "int", "num2": "int", ev: "int" }): linkL.append({ "source": flds["num1"], "target": flds["num2"], "value": flds[ev] }) links = json.JSONEncoder().encode(linkL) nolabel = "" if nl: nolabel = "font-size: 0px;" html = sys.stdout if not oFile == None: html = open(oFile, "w") outTemplate = ''' <!DOCTYPE html> <html class="ocks-org do-not-copy"> <meta charset="utf-8"> <title>{title}</title> <style> body {{ font: 10px sans-serif; }} svg {{ padding: 10px 0 0 10px; }} .arc {{ stroke: #fff; }} #tooltip {{ position: absolute; width: 150px; height: auto; padding: 10px; background-color: white; -webkit-border-radius: 10px; -moz-border-radius: 10px; border-radius: 10px; -webkit-box-shadow: 4px 4px 10px rgba(0,0,0,0.4); -moz-box-shadow: 4px 4px 10px rgba(0,0,0,0.4); box-shadow: 4px 4px 10px rgba(0,0,0,0.4); pointer-events: none; }} #tooltip.hidden {{ display: none; }} #tooltip p {{ margin: 0; font-family: sans-serif; font-size: 10px; line-height: 14px; }} #chart {{ height: 500px; }} .node rect {{ cursor: move; fill-opacity: .9; shape-rendering: crispEdges; }} .node text {{ pointer-events: none; text-shadow: 0 1px 0 #fff; {nolabel} }} .link {{ fill: none; stroke: #000; stroke-opacity: .2; }} .link:hover {{ stroke-opacity: .5; }} </style> <body> <h1>{title}</h1> <p id="chart"> <script> {d3js_str} d3.sankey = function() {{ var sankey = {{}}, nodeWidth = 24, nodePadding = 8, size = [1, 1], nodes = [], links = []; sankey.nodeWidth = function(_) {{ if (!arguments.length) return nodeWidth; nodeWidth = +_; return sankey; }}; sankey.nodePadding = function(_) {{ if (!arguments.length) return nodePadding; nodePadding = +_; return sankey; }}; sankey.nodes = function(_) {{ if (!arguments.length) return nodes; nodes = _; return sankey; }}; sankey.links = function(_) {{ if (!arguments.length) return links; links = _; return sankey; }}; sankey.size = function(_) {{ if (!arguments.length) return size; size = _; return sankey; }}; sankey.layout = function(iterations){{ computeNodeLinks(); computeNodeValues(); computeNodeBreadths(); computeNodeDepths(iterations); computeLinkDepths(); return sankey; }}; sankey.relayout = function() {{ computeLinkDepths(); return sankey; }}; sankey.link = function() {{ var curvature = .5; function link(d) {{ var x0 = d.source.x + d.source.dx, x1 = d.target.x, xi = d3.interpolateNumber(x0, x1), x2 = xi(curvature), x3 = xi(1 - curvature), y0 = d.source.y + d.sy + d.dy / 2, y1 = d.target.y + d.ty + d.dy / 2; return "M" + x0 + "," + y0 + "C" + x2 + "," + y0 + " " + x3 + "," + y1 + " " + x1 + "," + y1; }} link.curvature = function(_) {{ if (!arguments.length) return curvature; curvature = +_; return link; }}; return link; }}; // Populate the sourceLinks and targetLinks for each node. // Also, if the source and target are not objects, assume they are indices. function computeNodeLinks() {{ nodes.forEach(function(node) {{ node.sourceLinks = []; node.targetLinks = []; }}); links.forEach(function(link) {{ var source = link.source, target = link.target; if (typeof source === "number") source = link.source = nodes[link.source]; if (typeof target === "number") target = link.target = nodes[link.target]; source.sourceLinks.push(link); target.targetLinks.push(link); }}); }} // Compute the value (size) of each node by summing the associated links. function computeNodeValues() {{ nodes.forEach( function(node) {{ node.value = Math.max(d3.sum(node.sourceLinks, value),d3.sum(node.targetLinks, value)); }} ); }} // Iteratively assign the breadth (x-position) for each node. // Nodes are assigned the maximum breadth of incoming neighbors plus one; // nodes with no incoming links are assigned breadth zero, while // nodes with no outgoing links are assigned the maximum breadth. function computeNodeBreadths() {{ var remainingNodes = nodes, nextNodes, x = 0; while (remainingNodes.length) {{ nextNodes = []; remainingNodes.forEach(function(node) {{ node.x = x; node.dx = nodeWidth; node.sourceLinks.forEach(function(link) {{ nextNodes.push(link.target); }}); }}); remainingNodes = nextNodes; ++x; }} // moveSinksRight(x); scaleNodeBreadths((width - nodeWidth) / (x - 1)); }} function moveSourcesRight() {{ nodes.forEach(function(node) {{ if (!node.targetLinks.length) {{ node.x = d3.min(node.sourceLinks, function(d) {{ return d.target.x; }} ) - 1; }} }}); }} function moveSinksRight(x) {{ nodes.forEach(function(node) {{ if (!node.sourceLinks.length) {{ node.x = x - 1; }} }}); }} function scaleNodeBreadths(kx) {{ nodes.forEach(function(node) {{ node.x *= kx; }}); }} function computeNodeDepths(iterations) {{ var nodesByBreadth = d3.nest() .key(function(d) {{ return d.x; }}) .sortKeys(d3.ascending) .entries(nodes) .map(function(d) {{ return d.values; }}); // initializeNodeDepth(); resolveCollisions(); for (var alpha = 1; iterations > 0; --iterations){{ relaxRightToLeft(alpha *= .99); resolveCollisions(); relaxLeftToRight(alpha); resolveCollisions(); }} function initializeNodeDepth() {{ var ky = d3.min(nodesByBreadth, function(nodes) {{ return (size[1] - (nodes.length - 1) * nodePadding) / d3.sum(nodes, value); }}); nodesByBreadth.forEach(function(nodes) {{ nodes.forEach(function(node, i) {{ node.y = i; node.dy = node.value * ky; }}); }}); links.forEach(function(link) {{ link.dy = link.value * ky; }}); }} function relaxLeftToRight(alpha) {{ nodesByBreadth.forEach(function(nodes, breadth) {{ nodes.forEach(function(node) {{ if (node.targetLinks.length) {{ var y = d3.sum(node.targetLinks, weightedSource) / d3.sum(node.targetLinks, value); node.y += (y - center(node)) * alpha; }} }}); }}); function weightedSource(link) {{ return center(link.source) * link.value; }} }} function relaxRightToLeft(alpha) {{ nodesByBreadth.slice().reverse().forEach(function(nodes){{ nodes.forEach(function(node) {{ if (node.sourceLinks.length) {{ var y = d3.sum(node.sourceLinks, weightedTarget) / d3.sum(node.sourceLinks, value); node.y += (y - center(node)) * alpha; }} }}); }}); function weightedTarget(link) {{ return center(link.target) * link.value; }} }} function resolveCollisions() {{ nodesByBreadth.forEach(function(nodes) {{ var node, dy, y0 = 0, n = nodes.length, i; // Push any overlapping nodes down. nodes.sort(ascendingDepth); for (i = 0; i < n; ++i) {{ node = nodes[i]; dy = y0 - node.y; if (dy > 0) node.y += dy; y0 = node.y + node.dy + nodePadding; }} // If the bottommost node goes outside the bounds, push it back up. dy = y0 - nodePadding - size[1]; if (dy > 0) {{ y0 = node.y -= dy; // Push any overlapping nodes back up. for (i = n - 2; i >= 0; --i) {{ node = nodes[i]; dy = node.y + node.dy + nodePadding - y0; if (dy > 0) node.y -= dy; y0 = node.y; }} }} }}); }} function ascendingDepth(a, b) {{ return a.y - b.y; }} }} function computeLinkDepths() {{ nodes.forEach(function(node) {{ node.sourceLinks.sort(ascendingTargetDepth); node.targetLinks.sort(ascendingSourceDepth); }}); nodes.forEach(function(node) {{ var sy = 0, ty = 0; node.sourceLinks.forEach(function(link) {{ link.sy = sy; sy += link.dy; }}); node.targetLinks.forEach(function(link) {{ link.ty = ty; ty += link.dy; }}); }}); function ascendingSourceDepth(a, b) {{ return a.source.y - b.source.y; }} function ascendingTargetDepth(a, b) {{ return a.target.y - b.target.y; }} }} function center(node){{ return node.y + node.dy / 2; }} function value(link) {{ return link.value; }} return sankey; }}; </script> <script> var margin = {{top: 1, right: 1, bottom: 6, left: 1}}, width = {width} - margin.left - margin.right, height = {height} - margin.top - margin.bottom; var formatNumber = d3.format(",.0f"), format = function(d) {{ return formatNumber(d) + " TWh"; }}, color = d3.scale.category20(); var svg = d3.select("#chart").append("svg") .attr("width", width + margin.left + margin.right) .attr("height", height + margin.top + margin.bottom) .append("g") .attr("transform", "translate(" + margin.left + "," + margin.top + ")"); var sankey = d3.sankey() .nodeWidth(15) .nodePadding(10) .size([width, height]); var path = sankey.link(); var nodes={nodes} var links={links} sankey .nodes(nodes) .links(links) .layout(32); var link = svg.append("g").selectAll(".link") .data(links) .enter().append("path") .attr("class", "link") .attr("d", path) .style("stroke-width", function(d) {{ return Math.max(1, d.dy); }}) .sort(function(a, b) {{ return b.dy - a.dy; }}); link.append("title") .text(function(d) {{ return d.source.name + " → " + d.target.name + "" + format(d.value); }}); var node = svg.append("g").selectAll(".node") .data(nodes) .enter().append("g") .attr("class", "node") .attr("transform", function(d) {{ return "translate(" + d.x + "," + d.y + ")"; }}) .call( d3.behavior.drag() .origin(function(d) {{ return d; }}) .on("dragstart", function() {{ this.parentNode.appendChild(this); }}) .on("drag", dragmove) ); node.append("rect") .attr("height", function(d) {{ return d.dy; }}) .attr("width", sankey.nodeWidth() ) .style("fill", function(d) {{ return d.color = color(d.name.replace(/ .*/, "")); }}) .style("stroke", function(d) {{ return d3.rgb(d.color).darker(2); }}) .append("title") .text(function(d) {{ return d.name + "" + format(d.value); }}); node.append("text") .attr("x", -6) .attr("y", function(d) {{ return d.dy / 2; }}) .attr("dy", ".35em") .attr("text-anchor", "end") .attr("transform", null) .text(function(d) {{ return d.name; }}) .filter(function(d) {{ return d.x < width / 2; }}) .attr("x", 6 + sankey.nodeWidth()) .attr("text-anchor", "start"); function dragmove(d){{ d3.select(this) .attr("transform", "translate(" + d.x + "," + (d.y = Math.max(0, Math.min(height - d.dy, d3.event.y))) + ")"); sankey.relayout(); link.attr("d", path); }} </script> '''.format(title=title, nolabel=nolabel, d3js_str=vjs.ViewJs.d3jsMin(), nodes=nodes, links=links, width=w, height=h) html.write(outTemplate) if not oFile == None: html.close()
def enumerate(self, eArgs): tf = mtemp.Mtemp() # 最小サポートと最小サポート件数 if "minCnt" in eArgs and eArgs["minCnt"] != None: self.minCnt = int(eArgs["minCnt"]) self.minSup = float(self.minCnt) / float(self.db.size) else: self.minSup = float(eArgs["minSup"]) self.minCnt = int(self.minSup * float(self.db.size) + 0.99) # 最大サポートと最大サポート件数 self.maxCnt = None if ("maxCnt" in eArgs and eArgs["maxCnt"] != None) or ("maxSup" in eArgs and eArgs["maxSup"] != None): if "maxCnt" in eArgs and eArgs["maxCnt"] != None: self.maxCnt = int(eArgs["maxCnt"]) self.maxSup = float(self.maxCnt) / float(self.db.size) else: self.maxSup = float(eArgs["maxSup"]) self.maxCnt = int(self.maxSup * float(self.db.size) + 0.99) # 列挙パターン数上限が指定されれば、一度lcmを実行して最小サポートを得る if "top" in eArgs and eArgs["top"] != None: self.top = eArgs["top"] # 列挙パターン数上限が指定されれば、一度lcmを実行して最小サポートを得る if self.top and self.top > 0: xxtop = tf.file() extTake.lcmseq(type="Cf", K=str(self.top), i=self.file, sup="1", so=xxtop) with open(xxtop, "r") as rfile: self.minCnt = int(rfile.read().strip()) # lcm_seq出力ファイル lcmout = tf.file() # 頻出パターンがなかった場合、lcm出力ファイルが生成されないので # そのときのために空ファイルを生成しておいく。 with open(lcmout, "w") as efile: pass # lcm_seqのパラメータ設定と実行 params = {} if self.msgoff: params["type"] = "CIf_" else: params["type"] = "CIf" if self.maxCnt: params["U"] = str(self.maxCnt) if "minLen" in eArgs: params["l"] = str(eArgs["minLen"]) if 'maxLen' in eArgs: params["u"] = str(eArgs["maxLen"]) if 'gap' in eArgs: params["g"] = str(eArgs["gap"]) if 'win' in eArgs: params["G"] = str(eArgs["win"]) params["i"] = self.file params["sup"] = str(self.minCnt) params["o"] = lcmout # lcm_seq実行 #MCMD::msgLog("#{run}") if 'padding' in eArgs and eArgs[ "padding"]: # padding指定時は、0アイテムを出力しないlcm_seqを実行 extTake.lcmseq_zero(params) else: extTake.lcmseq(params) # パターンのサポートを計算しCSV出力する self.pFile = self.temp.file() items = self.db.items transl = self.temp.file() extTake.lcmtrans(lcmout, "p", transl) f = nm.mdelnull(f="pattern", i=transl) f <<= nm.mvreplace(vf="pattern", m=items.file, K=items.idFN, f=items.itemFN) f <<= nm.msetstr(v=self.db.size, a="total") f <<= nm.mcal(c='${count}/${total}', a="support") # サポートの計算 f <<= nm.mcut(f="pid,pattern,size,count,total,support") f <<= nm.msortf(f="support%nr", o=self.pFile) f.run() if self.outtf: # トランザクション毎に出現するシーケンスを書き出す #MCMD::msgLog("output tid-patterns ...") self.tFile = self.temp.file() xxw = tf.file() #Mtemp.new.name f = None f <<= nm.mcut(f=self.db.idFN, i=self.db.file) f <<= nm.muniq(k=self.db.idFN) f <<= nm.mnumber(S=0, a="__tid", q=True) f <<= nm.msortf(f="__tid", o=xxw) f.run() translt = self.temp.file() extTake.lcmtrans(lcmout, "t", translt) f = None f <<= nm.msortf(f="__tid", i=translt) f <<= nm.mjoin(k="__tid", m=xxw, f=self.db.idFN) f <<= nm.mcut(f=self.db.idFN + ",pid") f <<= nm.msortf(f=self.db.idFN + ",pid", o=self.tFile) f.run()
def enumerate(self, eArgs): pFiles = [] tFiles = [] tf = mtemp.Mtemp() for cName, posSize in self.db.clsNameRecSize.items(): negSize = self.db.traSize - posSize if "minGR" in eArgs: self.minGR = eArgs["minGR"] else: minProb = eArgs["minProb"] if ("minProb" in eArgs) else 0.5 if "uniform" in eArgs and eArgs["uniform"] == True: self.minGR = (minProb / (1 - minProb)) * ( self.db.clsSize - 1) # マニュアルの式(4) else: self.minGR = (minProb / (1 - minProb)) * ( float(negSize) / float(posSize)) # マニュアルの式(4) # 最小サポートと最小サポート件数 # s=0.05 # s=c1:0.05,c2:0.06 # S=10 # S=c1:10,c2:15 if "minCnt" in eArgs: if isinstance(eArgs["minCnt"], dict): self.minPos = eArgs["minCnt"][cName] else: self.minPos = eArgs["minCnt"] else: if isinstance(eArgs["minSup"], dict): self.minPos = int(eArgs["minSup"][cName] * float(posSize) + 0.99) else: self.minPos = int(eArgs["minSup"] * flost(posSize) + 0.99) # 最大サポートと最大サポート件数 if "maxCnt" in eArgs: if isinstance(eArgs["maxCnt"], dict): self.maxPos = eArgs["maxCnt"][cName] else: self.maxPos = eArgs["maxCnt"] elif "maxSup" in eArgs: if isinstance(eArgs["maxSup"], dict): self.maxPos = int(eArgs["maxSup"][cName] * float(posSize) + 0.99) else: self.maxPos = int(eArgs["maxSup"] * float(posSize) + 0.99) else: self.maxPos = None self.sigma[cName] = self.calSigma(self.minPos, self.minGR, posSize, negSize) # lcmのパラメータ設定と実行 # 頻出パターンがなかった場合、lcm出力ファイルが生成されないので # そのときのために空ファイルを生成しておいく。 lcmout = tf.file() # lcm出力ファイル with open(lcmout, "w") as efile: pass runPara = {} if self.msgoff: runPara["type"] = eArgs["type"] + "IA_" else: runPara["type"] = eArgs["type"] + "IA" #if self.maxPos: #rubyだとif @maxCntなってる(どこにも設定されてないので)動いてないはず if self.maxPos: runPara["U"] = self.maxPos if "minLen" in eArgs: runPara["l"] = str(eArgs["minLen"]) if "maxLen" in eArgs: runPara["u"] = str(eArgs["maxLen"]) runPara["w"] = self.weightFile[cName] runPara["i"] = self.file runPara["sup"] = str(self.sigma[cName]) runPara["o"] = lcmout # lcm実行 #MCMD::msgLog("#{run}") #TAKE::run_lcm(run) #print(self.sigma) #print(runPara) #MCMD::msgLog("output patterns to CSV file ...") extTake.lcm(runPara) pFiles.append(self.temp.file()) transle = tf.file() extTake.lcmtrans(lcmout, "e", transle) f = nm.mdelnull(f="pattern", i=transle) f <<= nm.mcal(c='round(${countN},1)', a="neg") f <<= nm.mcal(c='round(${countP}/%s,1)' % (self.posWeight[cName]), a="pos") f <<= nm.mdelnull(f="pattern") #いる? f <<= nm.msetstr(v=cName, a="class") f <<= nm.msetstr(v=posSize, a="posTotal") f <<= nm.msetstr(v=self.minGR, a="minGR") f <<= nm.mcut(f="class,pid,pattern,size,pos,neg,posTotal,minGR", o=pFiles[-1]) f.run() #s = nutil.mrecount(i=self.file) #MCMD::msgLog("the number of contrast patterns on class `#{cName}' enumerated is #{s}") if self.outtf: # トランザクション毎に出現するパターンを書き出す #MCMD::msgLog("output tid-patterns ...") tFiles.append(self.temp.file()) xxw = tf.file() xxw = nm.mcut(f=self.db.idFN, i=self.db.file) xxw <<= nm.muniq(k=self.db.idFN) xxw <<= nm.mnumber(S=0, a="__tid", q=True) translt = self.temp.file() extTake.lcmtrans(lcmout, "t", translt) f = nm.mjoin(k="__tid", m=xxw, f=self.db.idFN, i=translt) f <<= nm.msetstr(v=cName, a="class") f <<= nm.mcut(f=self.db.idFN + ",class,pid", o=tFiles[-1]) f.run() # クラス別のパターンとtid-pidファイルを統合して最終出力 self.pFile = self.temp.file() self.tFile = self.temp.file() # パターンファイル併合 xxpCat = tf.file() f = nm.mcat(i=",".join(pFiles)) f <<= nm.msortf(f="class,pid") f <<= nm.mnumber(s="class,pid", S=0, a="ppid", o=xxpCat) f.run() # パターンファイル計算 items = self.db.items f = nm.mcut(f="class,ppid:pid,pattern,size,pos,neg,posTotal,minGR", i=xxpCat) f <<= nm.msetstr(v=self.db.traSize, a="total") f <<= nm.mcal(c='${total}-${posTotal}', a="negTotal") # negのトータル件数 f <<= nm.mcal(c='${pos}/${posTotal}', a="support") # サポートの計算 f <<= nm.mcal( c= 'if(${neg}==0,1.797693135e+308,(${pos}/${posTotal})/(${neg}/${negTotal}))', a="growthRate") if "uniform" in eArgs and eArgs["uniform"] == True: f <<= nm.mcal( c='(${pos}/${posTotal})/(${pos}/${posTotal}+(%s-1)*${neg}/${negTotal})' % (self.db.clsSize), a="postProb") else: f <<= nm.mcal(c='${pos}/(${pos}+${neg})', a="postProb") f <<= nm.msel(c='${pos}>=%s&&${growthRate}>=${minGR}' % (self.minPos)) # minSupとminGRによる選択 f <<= nm.mvreplace(vf="pattern", m=items.file, K=items.idFN, f=items.itemFN) f <<= nm.mcut( f="class,pid,pattern,size,pos,neg,posTotal,negTotal,total,support,growthRate,postProb" ) f <<= nm.mvsort(vf="pattern") f <<= nm.msortf(f="class%nr,postProb%nr,pos%nr", o=self.pFile) f.run() # アイテムを包含している冗長なタクソノミを削除 if items.taxonomy: #MCMD::msgLog("reducing redundant rules in terms of taxonomy ...") ##ここは後で zdd = VSOP.constant(0) dt = nm.mcut(i=self.pFile, f="pattern") for fldVal in dt: zdd = zdd + VSOP.itemset(fldVal[0]) zdd = self.reduceTaxo(zdd, self.db.items) xxp1 = tf.file() xxp2 = tf.file() xxp3 = tf.file() zdd.csvout(xxp1) nm.mcut(nfni=True, f="1:pattern", i=xxp1).mvsort(vf="pattern").msortf(f="pattern", o=xxp2).run() nm.msortf(f="pattern", i=self.pFile).mcommon( k="pattern", m=xxp2).msortf(f="class%nr,postProb%nr,pos%nr", o=xxp3).run() shutil.move(xxp3, self.pFile) if self.outtf: # 列挙されたパターンを含むtraのみ選択するためのマスタ xxp4 = nm.mcut(f="class,pid", i=self.pFile) f = nm.mcat(i=",".join(tFiles)) f <<= nm.mjoin(k="class,pid", m=xxpCat, f="ppid") # 全クラス統一pid(ppid)結合 f <<= nm.mcommon(k="class,ppid", K="class,pid", m=xxp4) # 列挙されたパターンの選択 f <<= nm.mcut(f=self.db.idFN + ",class,ppid:pid") f <<= nm.msortf(f=self.db.idFN + ",class,pid", o=self.tFile) f.run()
def __mkNode(key, nf, nl, nv, nc, ni, ef1, ef2, ei, noiso, norm, mapFile, oFile): xbyE = None xbyN = None # edgeファイルからnode情報を生成 # noiso(孤立node排除)の場合は、edgeにあってnodeにないidを省く必要があるので計算する。 if ni == None or (ni != None and noiso): inp = [ nm.mcut(f="%s:key,%s:nam,%s:nl" % (key, ef1, ef1), i=ei), nm.mcut(f="%s:key,%s:nam,%s:nl" % (key, ef2, ef2), i=ei) ] xbyE <<= nm.mnullto(i=inp, f="key", v="##NULL##") xbyE <<= nm.muniq(k="key,nam") xbyE <<= nm.mjoin(k="key", K="nam", m=mapFile, f="num:keyNum") xbyE <<= nm.mjoin(k="nam", K="nam", m=mapFile, f="num,leaf") xbyE <<= nm.msetstr(v=",,,,", a="nv,nc,nlKey,nvKey,ncKey") xbyE <<= nm.mcut( f="key,nam,keyNum,num,nl,nv,nc,leaf,nvKey,ncKey,nlKey") # nodeファイルから作成 if ni: # mcal cat用のlabel項目の作成 label = [] #label項目 if nl: for nml in nl: label.append(nml) else: label.append("$s{%s}" % (nf)) nvcStr = [] if nv: nvcStr.append('%s:nv' % (nv)) if nc: nvcStr.append('%s:nc' % (nc)) """ # map # nam,leaf,num # ##NULL##,,0 # #1_1,,2 # #1_2,,3 # #1_3,,4 # #2_1,,5 # a,1,6 # b,1,7 # c,1,8 """ f = None f <<= nm.mcal(c='cat("_",%s)' % (','.join(label)), a="##label", i=ni) if len(nvcStr) == 0: f <<= nm.mcut(f='%s:key,%s:nam,##label:nl' % (key, nf)) else: f <<= nm.mcut(f='%s:key,%s:nam,##label:nl,%s' % (key, nf, ','.join(nvcStr))) f <<= nm.mnullto(f="key", v="##NULL##") if not nv: f <<= nm.msetstr(v="", a="nv") if not nc: f <<= nm.msetstr(v="", a="nc") f <<= nm.mjoin(k="key", K="nam", m=mapFile, f="num:keyNum") f <<= nm.mjoin(k="nam", K="nam", m=mapFile, f="num,leaf") if norm: #ノードの拡大率は固定 nr = 3 f <<= nm.mnormalize(f="nv:nv2", c="range") f <<= nm.mcal(c='${nv2}*(%s-1)+1' % (nr), a="nvv") f <<= nm.mcut(f="key,nam,keyNum,num,nl,nvv:nv,nc,leaf") #o=#{xxa}" else: f <<= nm.mcut(f="key,nam,keyNum,num,nl,nv,nc,leaf") #o=#{xxa}" xbyN <<= nm.mjoin(k="keyNum", K="num", m=f, f="nl:nlk,nv:nvKey,nc:ncKey", n=True, i=f) xbyN <<= nm.mcal(c='if(isnull($s{nlk}),$s{key},$s{nlk})', a='nlKey') xbyN <<= nm.mcut(f="nlk", r=True) if ni != None and noiso: nm.mcommon(k="key,nam", m=xbyE, i=xbyN, o=oFile).run() elif ni != None: xbyN.writecsv(o=oFile).run() else: xbyE.writecsv(o=oFile).run()