def handleFile(phase, dirname, basename, suffix): filename = os.path.join(dirname, basename) #if the target file is older than the file this rule should be applied to, do nothing #Also check the rules dump - if this one is newer, then we have to execute this stuff, otherwise do nothing tFilename = os.path.join(const101.tRoot, dirname, basename + suffix) if not tools101.build(filename, tFilename) or not tools101.build(const101.rulesDump, tFilename): entry = {'units' : json.load(open(tFilename, 'r')), 'filename': filename} if not entry['units'] == []: matches.append(entry) return units = list() # metadata units for the file at hand id = 0 # current rule number for r in rules: rule = r["rule"] result = matchFile(phase, dirname, basename, rule) if not result is None: if "metadata" in rule: metadata = rule["metadata"] if isinstance(metadata, list): for m in metadata: buildUnit(units, id, m, result) else: buildUnit(units, id, metadata, result) id += 1 # Contract units to pay attention to dominators keys = list() removals = list() for unit in units: metadata = unit["metadata"] if "dominator" in metadata: keys.append(metadata["dominator"]) for key in keys: for unit in units: metadata = unit["metadata"] if key in metadata \ and (not "dominator" in metadata or not metadata["dominator"] == key): removals.append(unit) survivals = list() for unit in units: if not unit in removals: survivals.append(unit) units = survivals # Add entry to matches if any matches for file at hand tools101.makedirs(os.path.join(const101.tRoot, dirname)) matchesFile = open(tFilename, 'w') matchesFile.write(json.dumps(units)) if len(units) > 0: global noUnits global noFilesAffected noUnits += len(units) noFilesAffected += 1 entry = dict() entry["filename"] = filename entry["units"] = units matches.append(entry) tools101.tick()
def incProgress(): tools101.tick()
if debug: json.dump(commits, open('temporary.debug.json', 'w'), indent=4) print 'Finished gathering, will now associate commits with files' #map created list to real files for root, dirs, files in os.walk(const101.sRoot): for file in files: fileInfo = [] relPath = os.path.join(root,file).replace(const101.sRoot, '')[1:] for commit in commits: for change in commit['changes']: if change['file'] == relPath: fileInfo.append({ 'author': commit['author'], 'email': commit['email'], 'timestamp': commit['timestamp'], 'action' : change['action'] }) fileInfo = sorted(fileInfo, key=lambda k: k['timestamp']) if not os.path.exists(os.path.dirname(os.path.join(const101.tRoot,relPath))): os.makedirs(os.path.join(const101.tRoot,relPath)) json.dump(fileInfo, open(os.path.join(const101.tRoot,relPath+'.commitInfo.json'), 'w')) tools101.tick() print '\nFinished'
def incProgress(): tools101.tick();
def fun(dirname, dirs, files): tools101.tick() result = dict() result["languages"] = dict() result["technologies"] = dict() result["features"] = dict() result["concepts"] = dict() result["terms"] = dict() result["phrases"] = dict() result["refinedTokens"] = dict() # # Aggregation of file summaries # for basename in files: try: summaryFile = open(os.path.join(const101.tRoot, dirname, basename + ".summary.json"), 'r') summary = json.load(summaryFile) summaryFile.close() # Deal with languages for file addFile(result, "languages", "language", lambda x: x, basename, summary) # Deal with refinedTokens for file addRefinedTokens(result, dirname, basename, summary) # Deal with technologies for file addFile(result, "technologies", "partOf", lambda x: x, basename, summary) addFile(result, "technologies", "inputOf", lambda x: x, basename, summary) addFile(result, "technologies", "outputOf", lambda x: x, basename, summary) addFile(result, "technologies", "dependsOn", lambda x: x, basename, summary) # Deal with features for file addFile(result, "features", "feature", lambda x: x, basename, summary) # Deal with concepts for file addFile(result, "concepts", "concept", lambda x: x, basename, summary) # Deal with terms for file addFile(result, "terms", "term", lambda x: x, basename, summary) # Deal with phases for file addFile(result, "phrases", "phrase", phrase2str, basename, summary) except IOError: pass # # Aggregation of subdirectory indexes # for subdirname in dirs: subdirFile = open(os.path.join(const101.tRoot, dirname, subdirname, "index.json"), 'r') try: index = json.load(subdirFile) subdirFile.close() for key in result: if key == 'refinedTokens': addRefinedTokensDir(result, index) else: addDir(result, key, subdirname, index) except IOError: pass # Dumping the index result["dirs"] = dirs result["files"] = files resultFile = open(os.path.join(const101.tRoot, dirname, "index.json"), 'w') resultFile.write(json.dumps(result)) resultFile.close() # Dumping the file index fileresult = dict() fileresult["dirs"] = dirs fileresult["files"] = files fileresultFile = open(os.path.join(const101.tRoot, dirname, "fileindex.json"), 'w') fileresultFile.write(json.dumps(fileresult)) fileresultFile.close()