def handleFile(phase, dirname, basename, suffix):
filename = os.path.join(dirname, basename)
#if the target file is older than the file this rule should be applied to, do nothing
#Also check the rules dump - if this one is newer, then we have to execute this stuff, otherwise do nothing
tFilename = os.path.join(const101.tRoot, dirname, basename + suffix)
if not tools101.build(filename, tFilename) or not tools101.build(const101.rulesDump, tFilename):
entry = {'units' : json.load(open(tFilename, 'r')), 'filename': filename}
if not entry['units'] == []:
matches.append(entry)
return
units = list() # metadata units for the file at hand
id = 0 # current rule number
for r in rules:
rule = r["rule"]
result = matchFile(phase, dirname, basename, rule)
if not result is None:
if "metadata" in rule:
metadata = rule["metadata"]
if isinstance(metadata, list):
for m in metadata:
buildUnit(units, id, m, result)
else:
buildUnit(units, id, metadata, result)
id += 1
# Contract units to pay attention to dominators
keys = list()
removals = list()
for unit in units:
metadata = unit["metadata"]
if "dominator" in metadata:
keys.append(metadata["dominator"])
for key in keys:
for unit in units:
metadata = unit["metadata"]
if key in metadata \
and (not "dominator" in metadata
or not metadata["dominator"] == key):
removals.append(unit)
survivals = list()
for unit in units:
if not unit in removals:
survivals.append(unit)
units = survivals
# Add entry to matches if any matches for file at hand
tools101.makedirs(os.path.join(const101.tRoot, dirname))
matchesFile = open(tFilename, 'w')
matchesFile.write(json.dumps(units))
if len(units) > 0:
global noUnits
global noFilesAffected
noUnits += len(units)
noFilesAffected += 1
entry = dict()
entry["filename"] = filename
entry["units"] = units
matches.append(entry)
tools101.tick()
def incProgress():
tools101.tick()
if debug:
json.dump(commits, open('temporary.debug.json', 'w'), indent=4)
print 'Finished gathering, will now associate commits with files'
#map created list to real files
for root, dirs, files in os.walk(const101.sRoot):
for file in files:
fileInfo = []
relPath = os.path.join(root,file).replace(const101.sRoot, '')[1:]
for commit in commits:
for change in commit['changes']:
if change['file'] == relPath:
fileInfo.append({
'author': commit['author'],
'email': commit['email'],
'timestamp': commit['timestamp'],
'action' : change['action']
})
fileInfo = sorted(fileInfo, key=lambda k: k['timestamp'])
if not os.path.exists(os.path.dirname(os.path.join(const101.tRoot,relPath))):
os.makedirs(os.path.join(const101.tRoot,relPath))
json.dump(fileInfo, open(os.path.join(const101.tRoot,relPath+'.commitInfo.json'), 'w'))
tools101.tick()
print '\nFinished'
def incProgress():
tools101.tick();
def fun(dirname, dirs, files):
tools101.tick()
result = dict()
result["languages"] = dict()
result["technologies"] = dict()
result["features"] = dict()
result["concepts"] = dict()
result["terms"] = dict()
result["phrases"] = dict()
result["refinedTokens"] = dict()
#
# Aggregation of file summaries
#
for basename in files:
try:
summaryFile = open(os.path.join(const101.tRoot, dirname, basename + ".summary.json"), 'r')
summary = json.load(summaryFile)
summaryFile.close()
# Deal with languages for file
addFile(result, "languages", "language", lambda x: x, basename, summary)
# Deal with refinedTokens for file
addRefinedTokens(result, dirname, basename, summary)
# Deal with technologies for file
addFile(result, "technologies", "partOf", lambda x: x, basename, summary)
addFile(result, "technologies", "inputOf", lambda x: x, basename, summary)
addFile(result, "technologies", "outputOf", lambda x: x, basename, summary)
addFile(result, "technologies", "dependsOn", lambda x: x, basename, summary)
# Deal with features for file
addFile(result, "features", "feature", lambda x: x, basename, summary)
# Deal with concepts for file
addFile(result, "concepts", "concept", lambda x: x, basename, summary)
# Deal with terms for file
addFile(result, "terms", "term", lambda x: x, basename, summary)
# Deal with phases for file
addFile(result, "phrases", "phrase", phrase2str, basename, summary)
except IOError:
pass
#
# Aggregation of subdirectory indexes
#
for subdirname in dirs:
subdirFile = open(os.path.join(const101.tRoot, dirname, subdirname, "index.json"), 'r')
try:
index = json.load(subdirFile)
subdirFile.close()
for key in result:
if key == 'refinedTokens':
addRefinedTokensDir(result, index)
else:
addDir(result, key, subdirname, index)
except IOError:
pass
# Dumping the index
result["dirs"] = dirs
result["files"] = files
resultFile = open(os.path.join(const101.tRoot, dirname, "index.json"), 'w')
resultFile.write(json.dumps(result))
resultFile.close()
# Dumping the file index
fileresult = dict()
fileresult["dirs"] = dirs
fileresult["files"] = files
fileresultFile = open(os.path.join(const101.tRoot, dirname, "fileindex.json"), 'w')
fileresultFile.write(json.dumps(fileresult))
fileresultFile.close()
