def __process_functions(
functions: Iterable,
blob: str,
func_identifier_scope: Optional[str] = None
) -> Iterator[Dict[str, Any]]:
for function_node in functions:
if PythonParser.is_function_empty(function_node):
continue
function_metadata = PythonParser.get_function_metadata(
function_node, blob)
if func_identifier_scope is not None:
function_metadata['identifier'] = '{}.{}'.format(
func_identifier_scope, function_metadata['identifier'])
docstring_node = PythonParser.__get_docstring_node(function_node)
function_metadata['docstring'] = PythonParser.get_docstring(
docstring_node, blob)
function_metadata['docstring_summary'] = get_docstring_summary(
function_metadata['docstring'])
function_metadata['function'] = match_from_span(
function_node, blob)
function_metadata['function_tokens'] = tokenize_code(
function_node, blob)
function_metadata['start_point'] = function_node.start_point
function_metadata['end_point'] = function_node.end_point
yield function_metadata
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
functions = PythonParser.get_function_definitions(tree.root_node)
classes = PythonParser.get_class_definitions(tree.root_node)
definitions = list(PythonParser.__process_functions(functions, blob))
for _class in classes:
class_metadata = PythonParser.get_class_metadata(_class, blob)
docstring_node = PythonParser.__get_docstring_node(_class)
class_metadata['docstring'] = PythonParser.get_docstring(
docstring_node, blob)
class_metadata['docstring_summary'] = get_docstring_summary(
class_metadata['docstring'])
class_metadata['function'] = ''
class_metadata['function_tokens'] = []
class_metadata['start_point'] = _class.start_point
class_metadata['end_point'] = _class.end_point
definitions.append(class_metadata)
functions = PythonParser.get_function_definitions(_class)
definitions.extend(
PythonParser.__process_functions(functions, blob,
class_metadata['identifier']))
return definitions
def get_methods(module_or_class_node, blob: str) -> List[Dict[str, Any]]:
definitions = []
comment_buffer = []
module_or_class_name = match_from_span(module_or_class_node.children[1], blob)
for child in module_or_class_node.children:
if child.type == 'comment':
comment_buffer.append(child)
elif child.type == 'method':
docstring = '\n'.join([match_from_span(comment, blob).strip().strip('#') for comment in comment_buffer])
docstring_summary = get_docstring_summary(docstring)
metadata = RubyParser.get_function_metadata(child, blob)
if metadata['identifier'] in RubyParser.BLACKLISTED_FUNCTION_NAMES:
continue
definitions.append({
'type': 'class',
'identifier': '{}.{}'.format(module_or_class_name, metadata['identifier']),
'parameters': metadata['parameters'],
'function': match_from_span(child, blob),
'function_tokens': tokenize_code(child, blob),
'docstring': docstring,
'docstring_summary': docstring_summary,
'start_point': child.start_point,
'end_point': child.end_point
})
comment_buffer = []
else:
comment_buffer = []
return definitions
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
classes = (node for node in tree.root_node.children
if node.type == 'class_declaration')
definitions = []
for class_ in classes:
class_identifier = match_from_span([
child for child in class_.children
if child.type == 'identifier'
][0], blob).strip()
for child in (child for child in class_.children
if child.type == 'class_body'):
for idx, node in enumerate(child.children):
if node.type == 'method_declaration':
if JavaParser.is_method_body_empty(node):
continue
docstring = ''
if idx - 1 >= 0 and child.children[
idx - 1].type == 'comment':
docstring = match_from_span(
child.children[idx - 1], blob)
docstring = strip_c_style_comment_delimiters(
docstring)
docstring_summary = get_docstring_summary(docstring)
metadata = JavaParser.get_function_metadata(node, blob)
if metadata[
'identifier'] in JavaParser.BLACKLISTED_FUNCTION_NAMES:
continue
definitions.append({
'type':
node.type,
'identifier':
'{}.{}'.format(class_identifier,
metadata['identifier']),
'parameters':
metadata['parameters'],
'function':
match_from_span(node, blob),
'function_tokens':
tokenize_code(node, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
node.start_point,
'end_point':
node.end_point
})
return definitions
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
functions = []
function_nodes = []
traverse_type(tree.root_node, function_nodes, 'function_declaration')
method_nodes = []
traverse_type(tree.root_node, method_nodes, 'method_definition')
for declaration in function_nodes + method_nodes:
if declaration.children is None or len(declaration.children) == 0:
continue
functions.append(
(declaration,
JavascriptParser.get_docstring(tree, declaration, blob)))
definitions = []
for function_node, docstring in functions:
metadata = JavascriptParser.get_function_metadata(
function_node, blob)
docstring_summary = get_docstring_summary(docstring)
if metadata[
'identifier'] in JavascriptParser.BLACKLISTED_FUNCTION_NAMES:
continue
definitions.append({
'identifier':
metadata['identifier'],
'parameters':
metadata['parameters'],
'function':
match_from_span(function_node, blob),
'function_tokens':
tokenize_code(function_node, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
function_node.start_point,
'end_point':
function_node.end_point
})
return definitions
def get_declarations(declaration_node, blob: str,
node_type: str) -> List[Dict[str, Any]]:
declaration_lists = [
child for child in declaration_node.children
if child.type == 'declaration_list'
]
if len(declaration_lists) == 0:
return []
declaration_list = declaration_lists[0]
declaration_name = PhpParser.get_declaration_name(
declaration_node, blob)
declarations = []
for idx, child in enumerate(declaration_list.children):
if child.type == 'method_declaration':
docstring = PhpParser.get_docstring(declaration_list, blob,
idx)
docstring_summary = get_docstring_summary(docstring)
metadata = PhpParser.get_function_metadata(child, blob)
declarations.append({
'type':
child.type,
'identifier':
'{}.{}'.format(declaration_name, metadata['identifier']),
'parameters':
metadata['parameters'],
'function':
match_from_span(child, blob),
'function_tokens':
tokenize_code(child, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
child.start_point,
'end_point':
child.end_point
})
return declarations
def get_definitions(tree, blob: str) -> List[Dict[str, Any]]:
definitions = []
comment_buffer = []
for child in tree.root_node.children:
if child.type == 'comment':
comment_buffer.append(child)
elif child.type in ('method_declaration', 'function_declaration'):
docstring = '\n'.join([
match_from_span(comment, blob)
for comment in comment_buffer
])
docstring_summary = strip_c_style_comment_delimiters(
(get_docstring_summary(docstring)))
metadata = GoParser.get_function_metadata(child, blob)
definitions.append({
'type':
child.type,
'identifier':
metadata['identifier'],
'parameters':
metadata['parameters'],
'function':
match_from_span(child, blob),
'function_tokens':
tokenize_code(child, blob),
'docstring':
docstring,
'docstring_summary':
docstring_summary,
'start_point':
child.start_point,
'end_point':
child.end_point
})
comment_buffer = []
else:
comment_buffer = []
return definitions