def visitTerminal(self, node): curToken = node.symbol if curToken.type == -1: return if curToken.tokenIndex >= 1: # predict newline based upon curToken appearing after prevToken on same line prevToken = self.stream.tokens[curToken.tokenIndex - 1] curLine = prevToken.line curToken.column = prevToken.column + len(prevToken.text) else: curLine = 1 curToken.column = 0 curToken.line = curLine vars = groomlib.node_features(self.stream.tokens, node) feature_names = groomlib.FEATURE_NAMES n = len(feature_names) d = dict((feature_names[i], vars[i]) for i in range(0, n)) # print d transformed_features_testing = self.vec.transform(d).toarray() inject_newline = self.newline_forest.predict( transformed_features_testing) # newline_predictions_proba = self.forest.predict_proba(transformed_data_testing) # print curToken, "->", newline_predictions_proba # print "inject_newline", inject_newline if inject_newline: curToken.line += 1 curToken.column = 0 print # inject a newline # now figure out indent indent = self.indent_forest.predict(transformed_features_testing) # if indent!=0: # print "indent %d at %s" % (indent,curToken) self.current_indent += indent sys.stdout.write(" " * self.current_indent) # inject indent else: ws = self.whitespace_forest.predict(transformed_features_testing) sys.stdout.write(" " * ws) # inject whitespace before token sys.stdout.write(curToken.text)
def visitTerminal(self, node): curToken = node.symbol if curToken.type==-1: return if curToken.tokenIndex>=1: # predict newline based upon curToken appearing after prevToken on same line prevToken = self.stream.tokens[curToken.tokenIndex-1] curLine = prevToken.line curToken.column = prevToken.column + len(prevToken.text) else: curLine = 1 curToken.column = 0 curToken.line = curLine vars = groomlib.node_features(self.stream.tokens, node) feature_names = groomlib.FEATURE_NAMES n = len(feature_names) d = dict((feature_names[i], vars[i]) for i in range(0, n)) # print d transformed_features_testing = self.vec.transform(d).toarray() inject_newline = self.newline_forest.predict(transformed_features_testing) # newline_predictions_proba = self.forest.predict_proba(transformed_data_testing) # print curToken, "->", newline_predictions_proba # print "inject_newline", inject_newline if inject_newline: curToken.line += 1 curToken.column = 0 print # inject a newline # now figure out indent indent = self.indent_forest.predict(transformed_features_testing) # if indent!=0: # print "indent %d at %s" % (indent,curToken) self.current_indent += indent sys.stdout.write(" " * self.current_indent) # inject indent else: ws = self.whitespace_forest.predict(transformed_features_testing) sys.stdout.write(" " * ws) # inject whitespace before token sys.stdout.write(curToken.text)
def visitTerminal(self, node): i = node.symbol.tokenIndex curToken = node.symbol if curToken.type==-1: return precedingNL = False column_delta = 0 ws = 0 if i>=1: prevToken = self.stream.tokens[i-1] precedingNL = curToken.line > prevToken.line if precedingNL: if self.first_token_on_line is not None: column_delta = curToken.column - self.first_token_on_line.column self.first_token_on_line = curToken else: ws = curToken.column - (prevToken.column+len(prevToken.text)) #print dir(node) vars = groomlib.node_features(self.stream.tokens, node) self.inject_newlines.append(1 if precedingNL else 0) self.indent.append(column_delta) self.whitespace.append(ws) self.features.append(vars)
def visitTerminal(self, node): i = node.symbol.tokenIndex curToken = node.symbol if curToken.type == -1: return precedingNL = False column_delta = 0 ws = 0 if i >= 1: prevToken = self.stream.tokens[i - 1] precedingNL = curToken.line > prevToken.line if precedingNL: if self.first_token_on_line is not None: column_delta = curToken.column - self.first_token_on_line.column self.first_token_on_line = curToken else: ws = curToken.column - (prevToken.column + len(prevToken.text)) #print dir(node) vars = groomlib.node_features(self.stream.tokens, node) self.inject_newlines.append(1 if precedingNL else 0) self.indent.append(column_delta) self.whitespace.append(ws) self.features.append(vars)