def get_methods(module_or_class_node, blob: str) -> List[Dict[str, Any]]:
definitions = []
comment_buffer = []
module_or_class_name = match_from_span(module_or_class_node.children[1], blob)
for child in module_or_class_node.children:
if child.type == 'comment':
comment_buffer.append(child)
elif child.type == 'method':
docstring = '\n'.join([match_from_span(comment, blob).strip().strip('#') for comment in comment_buffer])
docstring_summary = get_docstring_summary(docstring)
metadata = RubyParser.get_function_metadata(child, blob)
if metadata['identifier'] in RubyParser.BLACKLISTED_FUNCTION_NAMES:
continue
definitions.append({
'type': 'class',
'identifier': '{}.{}'.format(module_or_class_name, metadata['identifier']),
'parameters': metadata['parameters'],
'function': match_from_span(child, blob),
'function_tokens': tokenize_code(child, blob),
'docstring': docstring,
'docstring_summary': docstring_summary,
'start_point': child.start_point,
'end_point': child.end_point
})
comment_buffer = []
else:
comment_buffer = []
return definitions
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
classes = (node for node in tree.root_node.children
if node.type == 'class_declaration')
definitions = []
for class_ in classes:
class_identifier = match_from_span([
child for child in class_.children
if child.type == 'identifier'
][0], blob).strip()
for child in (child for child in class_.children
if child.type == 'class_body'):
for idx, node in enumerate(child.children):
if node.type == 'method_declaration':
if JavaParser.is_method_body_empty(node):
continue
docstring = ''
if idx - 1 >= 0 and child.children[
idx - 1].type == 'comment':
docstring = match_from_span(
child.children[idx - 1], blob)
docstring = strip_c_style_comment_delimiters(
docstring)
docstring_summary = get_docstring_summary(docstring)
metadata = JavaParser.get_function_metadata(node, blob)
if metadata[
'identifier'] in JavaParser.BLACKLISTED_FUNCTION_NAMES:
continue
definitions.append({
'type':
node.type,
'identifier':
'{}.{}'.format(class_identifier,
metadata['identifier']),
'parameters':
metadata['parameters'],
'function':
match_from_span(node, blob),
'function_tokens':
tokenize_code(node, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
node.start_point,
'end_point':
node.end_point
})
return definitions
def get_docstring(tree, node, blob: str) -> str:
docstring = ''
parent_node = node_parent(tree, node)
if parent_node.type == 'variable_declarator':
base_node = node_parent(
tree, parent_node) # Get the variable declaration
elif parent_node.type == 'pair':
base_node = parent_node # This is a common pattern where a function is assigned as a value to a dictionary.
else:
base_node = node
prev_sibling = previous_sibling(tree, base_node)
if prev_sibling is not None and prev_sibling.type == 'comment':
all_prev_comment_nodes = [prev_sibling]
prev_sibling = previous_sibling(tree, prev_sibling)
while prev_sibling is not None and prev_sibling.type == 'comment':
all_prev_comment_nodes.append(prev_sibling)
last_comment_start_line = prev_sibling.start_point[0]
prev_sibling = previous_sibling(tree, prev_sibling)
if prev_sibling is not None and prev_sibling.end_point[
0] + 1 < last_comment_start_line:
break # if there is an empty line, stop expanding.
docstring = ' '.join(
(strip_c_style_comment_delimiters(match_from_span(s, blob))
for s in all_prev_comment_nodes[::-1]))
return docstring
def __process_functions(
functions: Iterable,
blob: str,
func_identifier_scope: Optional[str] = None
) -> Iterator[Dict[str, Any]]:
for function_node in functions:
if PythonParser.is_function_empty(function_node):
continue
function_metadata = PythonParser.get_function_metadata(
function_node, blob)
if func_identifier_scope is not None:
function_metadata['identifier'] = '{}.{}'.format(
func_identifier_scope, function_metadata['identifier'])
docstring_node = PythonParser.__get_docstring_node(function_node)
function_metadata['docstring'] = PythonParser.get_docstring(
docstring_node, blob)
function_metadata['docstring_summary'] = get_docstring_summary(
function_metadata['docstring'])
function_metadata['function'] = match_from_span(
function_node, blob)
function_metadata['function_tokens'] = tokenize_code(
function_node, blob)
function_metadata['start_point'] = function_node.start_point
function_metadata['end_point'] = function_node.end_point
yield function_metadata
def get_function_metadata(function_node, blob: str) -> Dict[str, str]:
metadata = {'identifier': '', 'parameters': '', 'return_statement': ''}
is_header = False
for child in function_node.children:
if is_header:
if child.type == 'identifier':
metadata['identifier'] = match_from_span(child, blob)
elif child.type == 'parameters':
metadata['parameters'] = match_from_span(child, blob)
if child.type == 'def':
is_header = True
elif child.type == ':':
is_header = False
elif child.type == 'return_statement':
metadata['return_statement'] = match_from_span(child, blob)
return metadata
def get_function_metadata(function_node, blob: str) -> Dict[str, str]:
metadata = {
'identifier': '',
'parameters': '',
}
parameters = []
for n in function_node.children:
if n.type == 'identifier':
metadata['identifier'] = match_from_span(n, blob).strip('(')
elif n.type == 'formal_parameters':
for fp_child in n.children:
if fp_child.type == 'formal_parameter':
parameters.append(match_from_span(fp_child, blob))
metadata['parameters'] = ' '.join(parameters)
return metadata
def get_class_metadata(class_node, blob: str) -> Dict[str, str]:
metadata = {
'identifier': '',
'argument_list': '',
}
is_header = False
for child in class_node.children:
if is_header:
if child.type == 'identifier':
metadata['identifier'] = match_from_span(child, blob)
elif child.type == 'argument_list':
metadata['argument_list'] = match_from_span(child, blob)
if child.type == 'class':
is_header = True
elif child.type == ':':
break
return metadata
def get_function_metadata(function_node, blob: str) -> Dict[str, str]:
metadata = {
'identifier': '',
'parameters': '',
}
if function_node.type == 'function_declaration':
metadata['identifier'] = match_from_span(function_node.children[1],
blob)
metadata['parameters'] = match_from_span(function_node.children[2],
blob)
elif function_node.type == 'method_declaration':
metadata['identifier'] = match_from_span(function_node.children[2],
blob)
metadata['parameters'] = ' '.join([
match_from_span(function_node.children[1], blob),
match_from_span(function_node.children[3], blob)
])
return metadata
def get_function_metadata(function_node, blob: str) -> Dict[str, str]:
metadata = {
'identifier': '',
'parameters': '',
}
identifier_nodes = [
child for child in function_node.children
if child.type in ['identifier', 'property_identifier']
]
formal_parameters_nodes = [
child for child in function_node.children
if child.type == 'formal_parameters'
]
if identifier_nodes:
metadata['identifier'] = match_from_span(identifier_nodes[0], blob)
if formal_parameters_nodes:
metadata['parameters'] = match_from_span(
formal_parameters_nodes[0], blob)
return metadata
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
definitions = []
comment_buffer = []
for child in tree.root_node.children:
if child.type == 'comment':
comment_buffer.append(child)
elif child.type in ('method_declaration', 'function_declaration'):
docstring = '\n'.join([
match_from_span(comment, blob)
for comment in comment_buffer
])
docstring_summary = strip_c_style_comment_delimiters(
(get_docstring_summary(docstring)))
metadata = GoParser.get_function_metadata(child, blob)
definitions.append({
'type':
child.type,
'identifier':
metadata['identifier'],
'parameters':
metadata['parameters'],
'function':
match_from_span(child, blob),
'function_tokens':
tokenize_code(child, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
child.start_point,
'end_point':
child.end_point
})
comment_buffer = []
else:
comment_buffer = []
return definitions
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
functions = []
function_nodes = []
traverse_type(tree.root_node, function_nodes, 'function_declaration')
method_nodes = []
traverse_type(tree.root_node, method_nodes, 'method_definition')
for declaration in function_nodes + method_nodes:
if declaration.children is None or len(declaration.children) == 0:
continue
functions.append(
(declaration,
JavascriptParser.get_docstring(tree, declaration, blob)))
definitions = []
for function_node, docstring in functions:
metadata = JavascriptParser.get_function_metadata(
function_node, blob)
docstring_summary = get_docstring_summary(docstring)
if metadata[
'identifier'] in JavascriptParser.BLACKLISTED_FUNCTION_NAMES:
continue
definitions.append({
'identifier':
metadata['identifier'],
'parameters':
metadata['parameters'],
'function':
match_from_span(function_node, blob),
'function_tokens':
tokenize_code(function_node, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
function_node.start_point,
'end_point':
function_node.end_point
})
return definitions
def get_declarations(declaration_node, blob: str,
node_type: str) -> List[Dict[str, Any]]:
declaration_lists = [
child for child in declaration_node.children
if child.type == 'declaration_list'
]
if len(declaration_lists) == 0:
return []
declaration_list = declaration_lists[0]
declaration_name = PhpParser.get_declaration_name(
declaration_node, blob)
declarations = []
for idx, child in enumerate(declaration_list.children):
if child.type == 'method_declaration':
docstring = PhpParser.get_docstring(declaration_list, blob,
idx)
docstring_summary = get_docstring_summary(docstring)
metadata = PhpParser.get_function_metadata(child, blob)
declarations.append({
'type':
child.type,
'identifier':
'{}.{}'.format(declaration_name, metadata['identifier']),
'parameters':
metadata['parameters'],
'function':
match_from_span(child, blob),
'function_tokens':
tokenize_code(child, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
child.start_point,
'end_point':
child.end_point
})
return declarations
def get_function_metadata(function_node, blob: str) -> Dict[str, str]:
metadata = {'identifier': match_from_span(function_node.children[1], blob), 'parameters': ''}
if function_node.children[2].type == 'method_parameters':
metadata['parameters'] = match_from_span(function_node.children[2], blob)
return metadata
def get_function_metadata(function_node, blob: str) -> Dict[str, str]:
metadata = {
'identifier': match_from_span(function_node.children[1], blob),
'parameters': match_from_span(function_node.children[2], blob)
}
return metadata
def get_declaration_name(declaration_node, blob: str):
for child in declaration_node.children:
if child.type == 'name':
return match_from_span(child, blob)
return ''
def get_docstring(docstring_node, blob: str) -> str:
docstring = ''
if docstring_node is not None:
docstring = match_from_span(docstring_node, blob)
docstring = docstring.strip().strip('"').strip("'")
return docstring
def get_docstring(trait_node, blob: str, idx: int) -> str:
docstring = ''
if idx - 1 >= 0 and trait_node.children[idx - 1].type == 'comment':
docstring = match_from_span(trait_node.children[idx - 1], blob)
docstring = strip_c_style_comment_delimiters(docstring)
return docstring
