def visitTerminal(self, node):
curToken = node.symbol
if curToken.type == -1:
return
if curToken.tokenIndex >= 1:
# predict newline based upon curToken appearing after prevToken on same line
prevToken = self.stream.tokens[curToken.tokenIndex - 1]
curLine = prevToken.line
curToken.column = prevToken.column + len(prevToken.text)
else:
curLine = 1
curToken.column = 0
curToken.line = curLine
vars = groomlib.node_features(self.stream.tokens, node)
feature_names = groomlib.FEATURE_NAMES
n = len(feature_names)
d = dict((feature_names[i], vars[i]) for i in range(0, n))
# print d
transformed_features_testing = self.vec.transform(d).toarray()
inject_newline = self.newline_forest.predict(
transformed_features_testing)
# newline_predictions_proba = self.forest.predict_proba(transformed_data_testing)
# print curToken, "->", newline_predictions_proba
# print "inject_newline", inject_newline
if inject_newline:
curToken.line += 1
curToken.column = 0
print # inject a newline
# now figure out indent
indent = self.indent_forest.predict(transformed_features_testing)
# if indent!=0:
# print "indent %d at %s" % (indent,curToken)
self.current_indent += indent
sys.stdout.write(" " * self.current_indent) # inject indent
else:
ws = self.whitespace_forest.predict(transformed_features_testing)
sys.stdout.write(" " * ws) # inject whitespace before token
sys.stdout.write(curToken.text)
def visitTerminal(self, node):
curToken = node.symbol
if curToken.type==-1:
return
if curToken.tokenIndex>=1:
# predict newline based upon curToken appearing after prevToken on same line
prevToken = self.stream.tokens[curToken.tokenIndex-1]
curLine = prevToken.line
curToken.column = prevToken.column + len(prevToken.text)
else:
curLine = 1
curToken.column = 0
curToken.line = curLine
vars = groomlib.node_features(self.stream.tokens, node)
feature_names = groomlib.FEATURE_NAMES
n = len(feature_names)
d = dict((feature_names[i], vars[i]) for i in range(0, n))
# print d
transformed_features_testing = self.vec.transform(d).toarray()
inject_newline = self.newline_forest.predict(transformed_features_testing)
# newline_predictions_proba = self.forest.predict_proba(transformed_data_testing)
# print curToken, "->", newline_predictions_proba
# print "inject_newline", inject_newline
if inject_newline:
curToken.line += 1
curToken.column = 0
print # inject a newline
# now figure out indent
indent = self.indent_forest.predict(transformed_features_testing)
# if indent!=0:
# print "indent %d at %s" % (indent,curToken)
self.current_indent += indent
sys.stdout.write(" " * self.current_indent) # inject indent
else:
ws = self.whitespace_forest.predict(transformed_features_testing)
sys.stdout.write(" " * ws) # inject whitespace before token
sys.stdout.write(curToken.text)
def visitTerminal(self, node):
i = node.symbol.tokenIndex
curToken = node.symbol
if curToken.type==-1:
return
precedingNL = False
column_delta = 0
ws = 0
if i>=1:
prevToken = self.stream.tokens[i-1]
precedingNL = curToken.line > prevToken.line
if precedingNL:
if self.first_token_on_line is not None:
column_delta = curToken.column - self.first_token_on_line.column
self.first_token_on_line = curToken
else:
ws = curToken.column - (prevToken.column+len(prevToken.text))
#print dir(node)
vars = groomlib.node_features(self.stream.tokens, node)
self.inject_newlines.append(1 if precedingNL else 0)
self.indent.append(column_delta)
self.whitespace.append(ws)
self.features.append(vars)
def visitTerminal(self, node):
i = node.symbol.tokenIndex
curToken = node.symbol
if curToken.type == -1:
return
precedingNL = False
column_delta = 0
ws = 0
if i >= 1:
prevToken = self.stream.tokens[i - 1]
precedingNL = curToken.line > prevToken.line
if precedingNL:
if self.first_token_on_line is not None:
column_delta = curToken.column - self.first_token_on_line.column
self.first_token_on_line = curToken
else:
ws = curToken.column - (prevToken.column + len(prevToken.text))
#print dir(node)
vars = groomlib.node_features(self.stream.tokens, node)
self.inject_newlines.append(1 if precedingNL else 0)
self.indent.append(column_delta)
self.whitespace.append(ws)
self.features.append(vars)
