def inheritmetadata(From,To,creates='../System/MetaData/'): FromPath = opmetadatapath(From) if IsDotPath(From) else metadatapath(From) ToPath = opmetadatapath(To) if IsDotPath(To) else metadatapath(To) X = ConsolidateSources(FromPath,extensions=['Associated','Attached','Inherited']) F = open(ToPath + '/InheritedMetaData.pickle','wb') pickle.dump(X,F) F.close() ProcessMetaData(ToPath,objname=To)
def NodeTypeDeterminer(n,NodeInfo,Mode,ShowUses,ShowImplied): ''' Technical dependency used in LabeledGraphFromLinks ''' N = NodeInfo[n] OpType = sum(N['OpType']) >= (1/2)*len(N) FunctionType = sum(N['FuncType']) >= (1/2)*len(N) OtherList = [OutsideFile(nn) for nn in N['Object']] Other = sum(OtherList) >= (1/2)*len(OtherList) ExistsList = [PathExists(nn) for nn in N['File']] Exists = all(ExistsList) if FunctionType: A = [(N['Object'][i],N['File'][i],N['FuncType'][i]) if ExistsList[i] else 'NotExists' for i in range(len(N)) if N['FuncType'][i]] else: A = [(N['Object'][i],N['File'][i],N['FuncType'][i]) if ExistsList[i] else 'NotExists' for i in range(len(N)) if not N['FuncType'][i]] B = list(set(A).difference(['NotExists'])) Path = 'NotExists' if len(B) == 0 else MaximalCommonPath([nn[1] if IsDir(nn[1]) else DirName(nn[1]) for nn in B]) if len(B) == 0: Summary = Fragment = Representation = 'NotExists' else: Summary = MaximalCommonPath([opmetadatapath(nn[0]) if nn[2] else metadatapath(nn[0]) for nn in B]) X = MaximalCommonPath([nn[0] for nn in B]) Y = MaximalCommonPath([nn[1] + ('@' + nn[0].split('.')[-1] if nn[2] else '') for nn in B]) Fragment = X if X != '' else Y.strip('@')[0] if Y!= '' else '../' Representation = Y if Y!= '' else '../' URL = '/.starflow/CGI-Executables/main_gui.cgi?Path=' + Path + '&Summary=' + Summary + '&Fragment=' + Fragment + '&Representation=' + Representation + '&Mode=' + Mode + '&ShowUses=' + ShowUses + '&ShowImplied=' + ShowImplied return [OpType,FunctionType,Exists,Other,URL]
def MakeLinkListHtml(LinkList,ClusterTags,inpath,outpath,Mode,ShowUses,ShowImplied): ''' takes a data dependency linklist and a list of cluster tags, and outputs an html representation of the linklist file (basically as a clickable html table) ARGUMENTS: --LinkList = the linklist to reresent, a numpy record array --ClusterTags = dictionary of clusters (output for example of the GetClusterTagDict function) --inpath = path that the LinkList is a representation of --outpath = place to store the resulting Html file --Mode = Name of the cluster collapse mode --ShowUses, ShowImplied are as in MakeLocalLinkGraph returns: nothing ''' LinkGen = lambda s,sp,sum,frag,rep,mode,su,si : '<a target = _top href="/.starflow/CGI-Executables/main_gui.cgi?Path=' + sp + '&Summary=' + sum + '&Fragment=' + frag + '&Representation=' + rep + '&Mode=' + mode + '&ShowUses=' + su + '&ShowImplied=' + si + '"/>' + s + '</a>' names = ['LinkType','LinkSource','SourceFile','LinkTarget','TargetFile','SourceCluster','TargetCluster'] Recs = [] for i in range(len(LinkList)): s = LinkList['LinkSource'][i] ; sp = LinkList['SourceFile'][i] if IsDir(LinkList['SourceFile'][i]) else DirName(LinkList['SourceFile'][i]) ssum = opmetadatapath(s) if LinkList['LinkType'][i] == 'Uses' or LinkList['LinkType'][i] == 'CreatedBy' else metadatapath(s) srep = LinkList['SourceFile'][i] + ('@' + LinkList['LinkSource'][i].split('.')[-1] if IsDotPath(LinkList['LinkSource'][i],LinkList['SourceFile'][i]) else '') spsum = metadatapath(sp) t = LinkList['LinkTarget'][i] ; tp = LinkList['TargetFile'][i] tsum = opmetadatapath(t) if LinkList['LinkType'][i] == 'Uses' or LinkList['LinkType'][i] == 'DependsOn' else metadatapath(t) trep = LinkList['TargetFile'][i] + ('@' + LinkList['LinkTarget'][i].split('.')[-1] if IsDotPath(LinkList['LinkTarget'][i],LinkList['TargetFile'][i]) else '') tpsum = metadatapath(tp) slink = LinkGen(s,sp,ssum,s,srep,Mode,ShowUses,ShowImplied) splink = LinkGen(sp,sp,spsum,sp,sp,Mode,ShowUses,ShowImplied) tlink = LinkGen(t,tp,tsum,t,trep,Mode,ShowUses,ShowImplied) tplink = LinkGen(tp,tp,tpsum,tp,tp,Mode,ShowUses,ShowImplied) Recs.append([LinkList['LinkType'][i],slink,splink,tlink,tplink,ClusterTags[s] if s in ClusterTags.keys() else '',ClusterTags[t] if t in ClusterTags.keys() else '']) RecList = numpy.rec.fromrecords(Recs,names=names) RecList.sort(order=['SourceCluster','TargetCluster','LinkSource','LinkTarget']) tb.web.tabular2html(fname=outpath,X = RecList,title = 'Link Graph for ' + inpath,split=False)
def generate_metadata(objname,forced=False,use=100): metapath = opmetadatapath(objname) if IsDotPath(objname) else metadatapath(objname) if IsDotPath(objname): path = '../' + '/'.join(objname.split('.')[:-1]) + '.py' objectname = objname.split('.')[-1] else: path = objname objectname ='' if forced or not PathExists(metapath) or os.path.getmtime(metapath) <= FindMtime(path,objectname=objectname,Simple=False): if IsDir(objname): if objname[-1] != '/': objname += '/' if is_hsv_dir(objname): pass else: D = {} L = [objname + ll for ll in listdir(objname) if not ll.startswith('.')] for l in L: D.update(generate_metadata(l,forced=forced)) LL = set(L).intersection(D.keys()) D[objname] = IntegrateDirMetaData([D[l] for l in LL]) return D else: if IsPythonFile(objname) or IsDotPath(objname): d = StoredDocstring(objname) if d: return {objname:{'description':d,'signature': 'python'}} elif objname.endswith(('.csv','.tsv')): if IsFile(objname): try: x = tabularmetadata(objname,use=use) except: x = DEFAULT_GenerateAutomaticMetaData(objname) print 'Failed to tabular metadata for', objname print_exc() else: x['signature'] = 'tabular' return {objname : x} return {} else: try: return {objname:pickle.load(open(metapath+'/AutomaticMetaData.pickle','r'))} except: return generate_metadata(objname,forced=True)
def MetaPathDir(Path): if Path.startswith('../') or OutsideFile(Path): return Backslash(metadatapath(Path)) else: return Backslash(opmetadatapath(Path))
def GetClusterTagDict(Path,LinkList,Mode): ''' Given a LinkList and a mode for collapsing the Linklist, produce a dictionary of node cluster tags. ARGUMENTS: Path -- path that the LinkList corresponds to LinkList -- list of Links as a numy record array Mode -- Mode by which to cluster the link nodes RETURNS: ClusterDict, a dictionary in which: -- the keys are (some of the) link sources or targets in the LinkList (e.g. potential nodes in the graph of the linklist) -- value on a key is a "cluster name " which represents a set of nodes that will collapsed together The "mode" determines which collapsing algorithm should be used. ''' if Mode == 'All': #no collapsing return {} elif Mode == 'ColDir': #collapse files within directory (except for files/subdirectories in 'Path', which are all not collapsed) t = time.time() if IsDir(Path): LinkList = LinkList.copy() K = uniqify(LinkList['LinkSource'].tolist() + LinkList['LinkTarget'].tolist()) OutSides = numpy.array([k for k in K if OutsideFile(k)]) ClusterDict = {} OutSides.sort() NotOutSide = numpy.invert(fastisin(LinkList['LinkSource'],OutSides)) LinkList = LinkList[NotOutSide] splitpath = Path.split('/') L = [] for j in range(1,len(splitpath)): p = '/'.join(splitpath[:j]) L += [p + '/' + l + ('/' if IsDir(p + '/' + l) else '') for l in os.listdir(p) if p + l != '/'.join(splitpath[:j+1])] L += [Backslash(Path) + l + ('/' if IsDir(Backslash(Path) + l) else '') for l in os.listdir(Path)] K = uniqify(LinkList['LinkSource'].tolist() + LinkList['LinkTarget'].tolist()) #ClusterDict.update(dict([(k,[l for l in L if SPathAlong(k,l)][-1]) for k in K if len([l for l in L if SPathAlong(k,l)]) > 0])) ClusterDict = {} if Path.startswith('../'): ModK = [k for k in K if k.startswith('../')] L = numpy.array(L) C = maximalpathalong(ModK,L) ClusterDict.update(dict([(ModK[i],C[i]) for i in range(len(ModK)) if C[i] != ''])) else: K = [k for k in K if not k.startswith('../')] ModK = [k.replace('.','/') for k in K] L = numpy.array(L) C = maximalpathalong(ModK,L) ClusterDict.update(dict([(K[i],C[i]) for i in range(len(K)) if C[i] != ''])) LinkList.sort(order=['LinkSource']) LS = LinkList.copy() LSDiffs = numpy.append(numpy.append([-1],(LS['LinkSource'][1:] != LS['LinkSource'][:-1]).nonzero()[0]),[len(LS)]) LinkList.sort(order=['LinkTarget']) LT = LinkList.copy() LTDiffs = numpy.append(numpy.append([-1],(LT['LinkTarget'][1:] != LT['LinkTarget'][:-1]).nonzero()[0]),[len(LT)]) for ii in range(2): SClusterDict2 = dict([(LS['LinkSource'][LSDiffs[i]],[ClusterDict[l] for l in LS['LinkTarget'][LSDiffs[i]:LSDiffs[i+1]] if l in ClusterDict.keys()]) for i in range(len(LSDiffs)-1) if LS['LinkSource'][LSDiffs[i]] not in ClusterDict.keys()]) TClusterDict2 = dict([(LT['LinkTarget'][LTDiffs[i]],[ClusterDict[l] for l in LT['LinkTarget'][LTDiffs[i]:LTDiffs[i+1]] if l in ClusterDict.keys()]) for i in range(len(LTDiffs)-1) if LT['LinkTarget'][LTDiffs[i]] not in ClusterDict.keys()]) ClusterDict2 = ProcessTwoDicts(SClusterDict2,TClusterDict2) C2K = numpy.array(ClusterDict2.keys()) C2V = numpy.array(ClusterDict2.values()) s = C2V.argsort() C2V = C2V[s] C2K = C2K[s] Diffs = numpy.append(numpy.append([-1],(C2V[1:] != C2V[:-1]).nonzero()[0]),[len(C2V)]) NewTags = dict(zip(C2K.tolist(),ListUnion([[MaximalCP(C2K[Diffs[i]+1:Diffs[i+1]+1])]*(Diffs[i+1] - Diffs[i]) for i in range(len(Diffs)-1)]))) ClusterDict.update(NewTags) if len(OutSides) > .1*len(uniqify(ClusterDict.values())) and len(OutSides) > 4: ClusterDict.update(dict([(k,'External Files') for k in OutSides])) return ClusterDict else: return {} elif Mode == 'ColPro': #collapse based on protocol tags ClusterDict = {} Targets = uniqify(LinkList['LinkTarget'].tolist() + LinkList['LinkSource'].tolist()) for target in Targets: metapaths = [opmetadatapath(target) + '/AssociatedMetaData.pickle',opmetadatapath(target) + '/AttachedMetaData.pickle'] for metapath in metapaths: if PathExists(metapath): MetaData = pickle.load(open(metapath,'r')) if isinstance(MetaData,dict) and 'ProtocolTags' in MetaData.keys(): ProtocolTags = MetaData['ProtocolTags'] if isinstance(ProtocolTags,dict): ClusterDict.update(ProtocolTags) return ClusterDict else: print 'No graph plotting mode selected.' return {}