def walkOverContribution(contributionPath, ignoreFunc = None, endsWith = '.tokens.json'):
result = {}
for (path, dirs, files) in os.walk(contributionPath):
if ignoreFunc(path):
continue
for file in files:
if file.endswith(endsWith):
if ignoreFunc(file):
continue
result[file] = Tokenization.tokenizeFile(os.path.join(path,file))
return result
def refineTokens(data, debug = False, force = True):
#find all .tokens.json files
files = Helper.derivedFiles(Helper.relevantFiles(data['data']), inputFileExt)
if (not force):
files = Helper.disregardFiles(files, inputFileExt, outputFileExt)
for file in files:
tokenized = Tokenization.tokenizeFile(file)
if (debug):
json.dump(tokenized, open(file.replace(inputFileExt, outputDebugFileExt), 'w'))
map = createMap(tokenized)
json.dump(map, open(file.replace('.tokens.json', outputFileExt), 'w'))
Helper.incProgress()
print ''
def refineTokens(data, debug=False, force=True):
#find all .tokens.json files
files = Helper.derivedFiles(Helper.relevantFiles(data['data']),
inputFileExt)
if (not force):
files = Helper.disregardFiles(files, inputFileExt, outputFileExt)
for file in files:
tokenized = Tokenization.tokenizeFile(file)
if (debug):
json.dump(
tokenized,
open(file.replace(inputFileExt, outputDebugFileExt), 'w'))
map = createMap(tokenized)
json.dump(map, open(file.replace('.tokens.json', outputFileExt), 'w'))
Helper.incProgress()
print ''
def refineTokens(debug = False, force = True, fragments=False):
#find all .tokens.json or fragments.tokens.json files
if fragments:
ending = inputFragFileExt
outEnding = outputFragFileExt
files = Helper.derivedFiles(Helper.relevantFiles(), inputFragFileExt)
if (not force):
files = Helper.disregardFiles(files, inputFragFileExt, outputFragFileExt)
else:
ending = inputFileExt
outEnding = outputFileExt
files = Helper.derivedFiles(Helper.relevantFiles(), inputFileExt)
if (not force):
files = Helper.disregardFiles(files, inputFileExt, outputFileExt)
for file in files:
tokenized = Tokenization.tokenizeFile(file, fragments=fragments)
if (debug):
json.dump(tokenized, open(file.replace(ending, outputDebugFileExt), 'w'))
map = createMap(tokenized, fragments=fragments)
json.dump(map, open(file.replace(ending, outEnding), 'w'))
Helper.incProgress()
print ''
