def inheritmetadata(From,To,creates='../System/MetaData/'): FromPath = opmetadatapath(From) if IsDotPath(From) else metadatapath(From) ToPath = opmetadatapath(To) if IsDotPath(To) else metadatapath(To) X = ConsolidateSources(FromPath,extensions=['Associated','Attached','Inherited']) F = open(ToPath + '/InheritedMetaData.pickle','wb') pickle.dump(X,F) F.close() ProcessMetaData(ToPath,objname=To)
def NodeTypeDeterminer(n,NodeInfo,Mode,ShowUses,ShowImplied):
'''
Technical dependency used in LabeledGraphFromLinks
'''
N = NodeInfo[n]
OpType = sum(N['OpType']) >= (1/2)*len(N)
FunctionType = sum(N['FuncType']) >= (1/2)*len(N)
OtherList = [OutsideFile(nn) for nn in N['Object']]
Other = sum(OtherList) >= (1/2)*len(OtherList)
ExistsList = [PathExists(nn) for nn in N['File']]
Exists = all(ExistsList)
if FunctionType:
A = [(N['Object'][i],N['File'][i],N['FuncType'][i]) if ExistsList[i] else 'NotExists' for i in range(len(N)) if N['FuncType'][i]]
else:
A = [(N['Object'][i],N['File'][i],N['FuncType'][i]) if ExistsList[i] else 'NotExists' for i in range(len(N)) if not N['FuncType'][i]]
B = list(set(A).difference(['NotExists']))
Path = 'NotExists' if len(B) == 0 else MaximalCommonPath([nn[1] if IsDir(nn[1]) else DirName(nn[1]) for nn in B])
if len(B) == 0:
Summary = Fragment = Representation = 'NotExists'
else:
Summary = MaximalCommonPath([opmetadatapath(nn[0]) if nn[2] else metadatapath(nn[0]) for nn in B])
X = MaximalCommonPath([nn[0] for nn in B])
Y = MaximalCommonPath([nn[1] + ('@' + nn[0].split('.')[-1] if nn[2] else '') for nn in B])
Fragment = X if X != '' else Y.strip('@')[0] if Y!= '' else '../'
Representation = Y if Y!= '' else '../'
URL = '/.starflow/CGI-Executables/main_gui.cgi?Path=' + Path + '&Summary=' + Summary + '&Fragment=' + Fragment + '&Representation=' + Representation + '&Mode=' + Mode + '&ShowUses=' + ShowUses + '&ShowImplied=' + ShowImplied
return [OpType,FunctionType,Exists,Other,URL]
def MakeLinkListHtml(LinkList,ClusterTags,inpath,outpath,Mode,ShowUses,ShowImplied):
'''
takes a data dependency linklist and a list of cluster tags, and
outputs an html representation of the linklist file (basically as
a clickable html table)
ARGUMENTS:
--LinkList = the linklist to reresent, a numpy record array
--ClusterTags = dictionary of clusters (output for example of
the GetClusterTagDict function)
--inpath = path that the LinkList is a representation of
--outpath = place to store the resulting Html file
--Mode = Name of the cluster collapse mode
--ShowUses, ShowImplied are as in MakeLocalLinkGraph
returns:
nothing
'''
LinkGen = lambda s,sp,sum,frag,rep,mode,su,si : '<a target = _top href="/.starflow/CGI-Executables/main_gui.cgi?Path=' + sp + '&Summary=' + sum + '&Fragment=' + frag + '&Representation=' + rep + '&Mode=' + mode + '&ShowUses=' + su + '&ShowImplied=' + si + '"/>' + s + '</a>'
names = ['LinkType','LinkSource','SourceFile','LinkTarget','TargetFile','SourceCluster','TargetCluster']
Recs = []
for i in range(len(LinkList)):
s = LinkList['LinkSource'][i] ; sp = LinkList['SourceFile'][i] if IsDir(LinkList['SourceFile'][i]) else DirName(LinkList['SourceFile'][i])
ssum = opmetadatapath(s) if LinkList['LinkType'][i] == 'Uses' or LinkList['LinkType'][i] == 'CreatedBy' else metadatapath(s)
srep = LinkList['SourceFile'][i] + ('@' + LinkList['LinkSource'][i].split('.')[-1] if IsDotPath(LinkList['LinkSource'][i],LinkList['SourceFile'][i]) else '')
spsum = metadatapath(sp)
t = LinkList['LinkTarget'][i] ; tp = LinkList['TargetFile'][i]
tsum = opmetadatapath(t) if LinkList['LinkType'][i] == 'Uses' or LinkList['LinkType'][i] == 'DependsOn' else metadatapath(t)
trep = LinkList['TargetFile'][i] + ('@' + LinkList['LinkTarget'][i].split('.')[-1] if IsDotPath(LinkList['LinkTarget'][i],LinkList['TargetFile'][i]) else '')
tpsum = metadatapath(tp)
slink = LinkGen(s,sp,ssum,s,srep,Mode,ShowUses,ShowImplied)
splink = LinkGen(sp,sp,spsum,sp,sp,Mode,ShowUses,ShowImplied)
tlink = LinkGen(t,tp,tsum,t,trep,Mode,ShowUses,ShowImplied)
tplink = LinkGen(tp,tp,tpsum,tp,tp,Mode,ShowUses,ShowImplied)
Recs.append([LinkList['LinkType'][i],slink,splink,tlink,tplink,ClusterTags[s] if s in ClusterTags.keys() else '',ClusterTags[t] if t in ClusterTags.keys() else ''])
RecList = numpy.rec.fromrecords(Recs,names=names)
RecList.sort(order=['SourceCluster','TargetCluster','LinkSource','LinkTarget'])
tb.web.tabular2html(fname=outpath,X = RecList,title = 'Link Graph for ' + inpath,split=False)
def generate_metadata(objname,forced=False,use=100):
metapath = opmetadatapath(objname) if IsDotPath(objname) else metadatapath(objname)
if IsDotPath(objname):
path = '../' + '/'.join(objname.split('.')[:-1]) + '.py'
objectname = objname.split('.')[-1]
else:
path = objname
objectname =''
if forced or not PathExists(metapath) or os.path.getmtime(metapath) <= FindMtime(path,objectname=objectname,Simple=False):
if IsDir(objname):
if objname[-1] != '/': objname += '/'
if is_hsv_dir(objname):
pass
else:
D = {}
L = [objname + ll for ll in listdir(objname) if not ll.startswith('.')]
for l in L:
D.update(generate_metadata(l,forced=forced))
LL = set(L).intersection(D.keys())
D[objname] = IntegrateDirMetaData([D[l] for l in LL])
return D
else:
if IsPythonFile(objname) or IsDotPath(objname):
d = StoredDocstring(objname)
if d:
return {objname:{'description':d,'signature': 'python'}}
elif objname.endswith(('.csv','.tsv')):
if IsFile(objname):
try:
x = tabularmetadata(objname,use=use)
except:
x = DEFAULT_GenerateAutomaticMetaData(objname)
print 'Failed to tabular metadata for', objname
print_exc()
else:
x['signature'] = 'tabular'
return {objname : x}
return {}
else:
try:
return {objname:pickle.load(open(metapath+'/AutomaticMetaData.pickle','r'))}
except:
return generate_metadata(objname,forced=True)
def MetaPathDir(Path):
if Path.startswith('../') or OutsideFile(Path):
return Backslash(metadatapath(Path))
else:
return Backslash(opmetadatapath(Path))
def GetClusterTagDict(Path,LinkList,Mode):
'''
Given a LinkList and a mode for collapsing the Linklist,
produce a dictionary of node cluster tags.
ARGUMENTS:
Path -- path that the LinkList corresponds to
LinkList -- list of Links as a numy record array
Mode -- Mode by which to cluster the link nodes
RETURNS:
ClusterDict, a dictionary in which:
-- the keys are (some of the) link sources or targets in the
LinkList (e.g. potential nodes in the graph of the linklist)
-- value on a key is a "cluster name " which represents a set
of nodes that will collapsed together
The "mode" determines which collapsing algorithm should be used.
'''
if Mode == 'All':
#no collapsing
return {}
elif Mode == 'ColDir':
#collapse files within directory (except for files/subdirectories in 'Path', which are all not collapsed)
t = time.time()
if IsDir(Path):
LinkList = LinkList.copy()
K = uniqify(LinkList['LinkSource'].tolist() + LinkList['LinkTarget'].tolist())
OutSides = numpy.array([k for k in K if OutsideFile(k)])
ClusterDict = {}
OutSides.sort()
NotOutSide = numpy.invert(fastisin(LinkList['LinkSource'],OutSides))
LinkList = LinkList[NotOutSide]
splitpath = Path.split('/')
L = []
for j in range(1,len(splitpath)):
p = '/'.join(splitpath[:j])
L += [p + '/' + l + ('/' if IsDir(p + '/' + l) else '') for l in os.listdir(p) if p + l != '/'.join(splitpath[:j+1])]
L += [Backslash(Path) + l + ('/' if IsDir(Backslash(Path) + l) else '') for l in os.listdir(Path)]
K = uniqify(LinkList['LinkSource'].tolist() + LinkList['LinkTarget'].tolist())
#ClusterDict.update(dict([(k,[l for l in L if SPathAlong(k,l)][-1]) for k in K if len([l for l in L if SPathAlong(k,l)]) > 0]))
ClusterDict = {}
if Path.startswith('../'):
ModK = [k for k in K if k.startswith('../')]
L = numpy.array(L)
C = maximalpathalong(ModK,L)
ClusterDict.update(dict([(ModK[i],C[i]) for i in range(len(ModK)) if C[i] != '']))
else:
K = [k for k in K if not k.startswith('../')]
ModK = [k.replace('.','/') for k in K]
L = numpy.array(L)
C = maximalpathalong(ModK,L)
ClusterDict.update(dict([(K[i],C[i]) for i in range(len(K)) if C[i] != '']))
LinkList.sort(order=['LinkSource'])
LS = LinkList.copy()
LSDiffs = numpy.append(numpy.append([-1],(LS['LinkSource'][1:] != LS['LinkSource'][:-1]).nonzero()[0]),[len(LS)])
LinkList.sort(order=['LinkTarget'])
LT = LinkList.copy()
LTDiffs = numpy.append(numpy.append([-1],(LT['LinkTarget'][1:] != LT['LinkTarget'][:-1]).nonzero()[0]),[len(LT)])
for ii in range(2):
SClusterDict2 = dict([(LS['LinkSource'][LSDiffs[i]],[ClusterDict[l] for l in LS['LinkTarget'][LSDiffs[i]:LSDiffs[i+1]] if l in ClusterDict.keys()]) for i in range(len(LSDiffs)-1) if LS['LinkSource'][LSDiffs[i]] not in ClusterDict.keys()])
TClusterDict2 = dict([(LT['LinkTarget'][LTDiffs[i]],[ClusterDict[l] for l in LT['LinkTarget'][LTDiffs[i]:LTDiffs[i+1]] if l in ClusterDict.keys()]) for i in range(len(LTDiffs)-1) if LT['LinkTarget'][LTDiffs[i]] not in ClusterDict.keys()])
ClusterDict2 = ProcessTwoDicts(SClusterDict2,TClusterDict2)
C2K = numpy.array(ClusterDict2.keys())
C2V = numpy.array(ClusterDict2.values())
s = C2V.argsort()
C2V = C2V[s]
C2K = C2K[s]
Diffs = numpy.append(numpy.append([-1],(C2V[1:] != C2V[:-1]).nonzero()[0]),[len(C2V)])
NewTags = dict(zip(C2K.tolist(),ListUnion([[MaximalCP(C2K[Diffs[i]+1:Diffs[i+1]+1])]*(Diffs[i+1] - Diffs[i]) for i in range(len(Diffs)-1)])))
ClusterDict.update(NewTags)
if len(OutSides) > .1*len(uniqify(ClusterDict.values())) and len(OutSides) > 4:
ClusterDict.update(dict([(k,'External Files') for k in OutSides]))
return ClusterDict
else:
return {}
elif Mode == 'ColPro':
#collapse based on protocol tags
ClusterDict = {}
Targets = uniqify(LinkList['LinkTarget'].tolist() + LinkList['LinkSource'].tolist())
for target in Targets:
metapaths = [opmetadatapath(target) + '/AssociatedMetaData.pickle',opmetadatapath(target) + '/AttachedMetaData.pickle']
for metapath in metapaths:
if PathExists(metapath):
MetaData = pickle.load(open(metapath,'r'))
if isinstance(MetaData,dict) and 'ProtocolTags' in MetaData.keys():
ProtocolTags = MetaData['ProtocolTags']
if isinstance(ProtocolTags,dict):
ClusterDict.update(ProtocolTags)
return ClusterDict
else:
print 'No graph plotting mode selected.'
return {}
