def walkOverContribution(contributionPath, ignoreFunc = None, endsWith = '.tokens.json'): result = {} for (path, dirs, files) in os.walk(contributionPath): if ignoreFunc(path): continue for file in files: if file.endswith(endsWith): if ignoreFunc(file): continue result[file] = Tokenization.tokenizeFile(os.path.join(path,file)) return result
def refineTokens(data, debug = False, force = True): #find all .tokens.json files files = Helper.derivedFiles(Helper.relevantFiles(data['data']), inputFileExt) if (not force): files = Helper.disregardFiles(files, inputFileExt, outputFileExt) for file in files: tokenized = Tokenization.tokenizeFile(file) if (debug): json.dump(tokenized, open(file.replace(inputFileExt, outputDebugFileExt), 'w')) map = createMap(tokenized) json.dump(map, open(file.replace('.tokens.json', outputFileExt), 'w')) Helper.incProgress() print ''
def refineTokens(data, debug=False, force=True): #find all .tokens.json files files = Helper.derivedFiles(Helper.relevantFiles(data['data']), inputFileExt) if (not force): files = Helper.disregardFiles(files, inputFileExt, outputFileExt) for file in files: tokenized = Tokenization.tokenizeFile(file) if (debug): json.dump( tokenized, open(file.replace(inputFileExt, outputDebugFileExt), 'w')) map = createMap(tokenized) json.dump(map, open(file.replace('.tokens.json', outputFileExt), 'w')) Helper.incProgress() print ''
def refineTokens(debug = False, force = True, fragments=False): #find all .tokens.json or fragments.tokens.json files if fragments: ending = inputFragFileExt outEnding = outputFragFileExt files = Helper.derivedFiles(Helper.relevantFiles(), inputFragFileExt) if (not force): files = Helper.disregardFiles(files, inputFragFileExt, outputFragFileExt) else: ending = inputFileExt outEnding = outputFileExt files = Helper.derivedFiles(Helper.relevantFiles(), inputFileExt) if (not force): files = Helper.disregardFiles(files, inputFileExt, outputFileExt) for file in files: tokenized = Tokenization.tokenizeFile(file, fragments=fragments) if (debug): json.dump(tokenized, open(file.replace(ending, outputDebugFileExt), 'w')) map = createMap(tokenized, fragments=fragments) json.dump(map, open(file.replace(ending, outEnding), 'w')) Helper.incProgress() print ''