def process_query(query, code): from parse import code_to_ast, ast_to_tree, tree_to_ast, parse import astor str_count = 0 str_map = dict() match_count = 1 match = QUOTED_STRING_RE.search(query) while match: str_repr = '_STR:%d_' % str_count str_literal = match.group(0) str_string = match.group(2) match_count += 1 # if match_count > 50: # return # query = QUOTED_STRING_RE.sub(str_repr, query, 1) str_map[str_literal] = str_repr str_count += 1 match = QUOTED_STRING_RE.search(query) code = code.replace(str_literal, '\'' + str_repr + '\'') # clean the annotation # query = query.replace('.', ' . ') for k, v in str_map.iteritems(): if k == '\'%s\'' or k == '\"%s\"': query = query.replace(v, k) code = code.replace('\'' + v + '\'', k) # tokenize query_tokens = nltk.word_tokenize(query) new_query_tokens = [] # break up function calls for token in query_tokens: new_query_tokens.append(token) i = token.find('.') if 0 < i < len(token) - 1: new_tokens = ['['] + token.replace('.', ' . ').split(' ') + [']'] new_query_tokens.extend(new_tokens) # check if the code compiles tree = parse(code) ast_tree = tree_to_ast(tree) astor.to_source(ast_tree) return new_query_tokens, code, str_map
def process_query(query, code): from parse import code_to_ast, ast_to_tree, tree_to_ast, parse import astor str_count = 0 str_map = dict() match_count = 1 match = QUOTED_STRING_RE.search(query) while match: str_repr = '_STR:%d_' % str_count str_literal = match.group(0) str_string = match.group(2) match_count += 1 # if match_count > 50: # return # query = QUOTED_STRING_RE.sub(str_repr, query, 1) str_map[str_literal] = str_repr str_count += 1 match = QUOTED_STRING_RE.search(query) code = code.replace(str_literal, '\'' + str_repr + '\'') # clean the annotation # query = query.replace('.', ' . ') for k, v in str_map.iteritems(): if k == '\'%s\'' or k == '\"%s\"': query = query.replace(v, k) code = code.replace('\'' + v + '\'', k) # tokenize query_tokens = nltk.word_tokenize(query) new_query_tokens = [] # break up function calls for token in query_tokens: new_query_tokens.append(token) i = token.find('.') if 0 < i < len(token) - 1: new_tokens = ['['] + token.replace('.', ' . ').split(' ') + [']'] new_query_tokens.extend(new_tokens) # check if the code compiles tree = parse(code) ast_tree = tree_to_ast(tree) astor.to_source(ast_tree) return new_query_tokens, code, str_map