def _ExtractTypedefTokens(self, doc_comment):
"""Yields typedef tokens from a DocComment."""
found_doc_flag = False
found_doc_start_brace = False
num_braces = 0
# We are looking for a @typedef token followed by an opening brace.
# We only emit from the opening brace to the last balanced closing brace.
for token in tokenutil.GetTokenRange(doc_comment.start_token,
doc_comment.end_token):
if token.type == Type.DOC_FLAG:
if token.string == '@typedef':
found_doc_flag = True
if not found_doc_flag:
continue
if token.type == Type.DOC_START_BRACE:
found_doc_start_brace = True
num_braces += 1
if not found_doc_start_brace:
continue
if token.type == Type.DOC_PREFIX:
continue
if token.type == Type.DOC_END_BRACE:
num_braces -= 1
if num_braces == 0:
yield token # Make sure we yield the final closing brace
return
yield token
def testGetTokenRange(self):
a = FakeToken()
b = FakeToken()
c = FakeToken()
d = FakeToken()
e = FakeToken()
a.next = b
b.next = c
c.next = d
self.assertEquals([a, b, c, d], tokenutil.GetTokenRange(a, d))
# This is an error as e does not come after a in the token chain.
self.assertRaises(Exception, lambda: tokenutil.GetTokenRange(a, e))
def _GetVarAssignmentTokens(context):
"""Returns the tokens from context if it is a var assignment.
Args:
context: An EcmaContext.
Returns:
If a var assignment, the tokens contained within it w/o the trailing
semicolon.
"""
if context.type != ecmametadatapass.EcmaContext.VAR:
return
# Get the tokens in this statement.
if context.start_token and context.end_token:
statement_tokens = tokenutil.GetTokenRange(context.start_token,
context.end_token)
else:
return
# And now just those tokens that are actually code.
is_non_code_type = lambda t: t.type not in JavaScriptTokenType.NON_CODE_TYPES
code_tokens = filter(is_non_code_type, statement_tokens)
# Pop off the semicolon if present.
if code_tokens and code_tokens[-1].IsType(JavaScriptTokenType.SEMICOLON):
code_tokens.pop()
if len(code_tokens) < 4:
return
if (code_tokens[0].IsKeyword('var') and
code_tokens[1].IsType(JavaScriptTokenType.SIMPLE_LVALUE) and
code_tokens[2].IsOperator('=')):
return code_tokens
def testFunctionParse(self):
functions, _ = testutil.ParseFunctionsAndComments(_FUNCTION_SCRIPT)
self.assertEquals(2, len(functions))
function = functions[0]
self.assertEquals(['aaa', 'bbb', 'ccc'], function.parameters)
start_token = function.start_token
end_token = function.end_token
# First function
self.assertEquals(
javascripttokens.JavaScriptTokenType.FUNCTION_DECLARATION,
function.start_token.type)
self.assertEquals('function', start_token.string)
self.assertEquals(3, start_token.line_number)
self.assertEquals(0, start_token.start_index)
self.assertEquals('}', end_token.string)
self.assertEquals(5, end_token.line_number)
self.assertEquals(0, end_token.start_index)
self.assertEquals('foo', function.name)
self.assertIsNone(function.doc)
function = functions[1]
self.assertEquals(['ddd', 'eee', 'fff'], function.parameters)
start_token = function.start_token
end_token = function.end_token
# Second function
self.assertEquals(
javascripttokens.JavaScriptTokenType.FUNCTION_DECLARATION,
function.start_token.type)
self.assertEquals('function', start_token.string)
self.assertEquals(11, start_token.line_number)
self.assertEquals(10, start_token.start_index)
self.assertEquals('}', end_token.string)
self.assertEquals(13, end_token.line_number)
self.assertEquals(0, end_token.start_index)
self.assertEquals('bar', function.name)
self.assertIsNotNone(function.doc)
# Check function JSDoc
doc = function.doc
doc_tokens = tokenutil.GetTokenRange(doc.start_token, doc.end_token)
comment_type = javascripttokens.JavaScriptTokenType.COMMENT
comment_tokens = filter(lambda t: t.type is comment_type, doc_tokens)
self.assertEquals('JSDoc comment.',
tokenutil.TokensToString(comment_tokens).strip())
def MatchAlias(context):
"""Match an alias statement (some identifier assigned to a variable).
Example alias: var MyClass = proj.longNamespace.MyClass.
Args:
context: An EcmaContext of type EcmaContext.STATEMENT.
Returns:
If a valid alias, returns a tuple of alias and symbol, otherwise None.
"""
if context.type != ecmametadatapass.EcmaContext.STATEMENT:
return
# Get the tokens in this statement.
if context.start_token and context.end_token:
statement_tokens = tokenutil.GetTokenRange(context.start_token,
context.end_token)
else:
return
# And now just those tokens that are actually code.
is_non_code_type = lambda t: t.type not in JavaScriptTokenType.NON_CODE_TYPES
code_tokens = filter(is_non_code_type, statement_tokens)
# This section identifies statements of the alias form "var alias = symbol".
# Pop off the semicolon if present.
if code_tokens and code_tokens[-1].IsType(JavaScriptTokenType.SEMICOLON):
code_tokens.pop()
if not (len(code_tokens) == 4 and code_tokens[0].IsKeyword('var') and
(code_tokens[0].metadata.context.type
== ecmametadatapass.EcmaContext.VAR)):
return
# Verify the only code tokens in this statement are part of the var
# declaration.
var_context = code_tokens[0].metadata.context
for token in code_tokens:
if token.metadata.context is not var_context:
return
# Verify that this is of the form "var lvalue = identifier;".
if not (code_tokens[0].IsKeyword('var')
and code_tokens[1].IsType(JavaScriptTokenType.SIMPLE_LVALUE)
and code_tokens[2].IsOperator('=')
and code_tokens[3].IsType(JavaScriptTokenType.IDENTIFIER)):
return
alias, symbol = code_tokens[1], code_tokens[3]
# Mark both tokens as an alias definition to avoid counting them as usages.
alias.metadata.is_alias_definition = True
symbol.metadata.is_alias_definition = True
return alias.string, symbol.string
def MatchAlias(context):
"""Match an alias statement (some identifier assigned to a variable).
Example alias: var MyClass = proj.longNamespace.MyClass.
Args:
context: An EcmaContext of type EcmaContext.VAR.
Returns:
If a valid alias, returns a tuple of alias and symbol, otherwise None.
"""
if context.type != ecmametadatapass.EcmaContext.VAR:
return
# The var's parent is a STATEMENT, which should be directly below goog.scope.
if not IsGoogScopeBlock(context.parent.parent):
return
# Get the tokens in this statement.
if context.start_token and context.end_token:
statement_tokens = tokenutil.GetTokenRange(context.start_token,
context.end_token)
else:
return
# And now just those tokens that are actually code.
is_non_code_type = lambda t: t.type not in JavaScriptTokenType.NON_CODE_TYPES
code_tokens = filter(is_non_code_type, statement_tokens)
# This section identifies statements of the alias form "var alias = symbol".
# Pop off the semicolon if present.
if code_tokens and code_tokens[-1].IsType(JavaScriptTokenType.SEMICOLON):
code_tokens.pop()
if len(code_tokens) < 4:
return
# Verify that this is of the form "var lvalue = identifier;".
# The identifier may span multiple lines and could be multiple tokens.
if (code_tokens[0].IsKeyword('var')
and code_tokens[1].IsType(JavaScriptTokenType.SIMPLE_LVALUE)
and code_tokens[2].IsOperator('=') and all(
t.IsType(JavaScriptTokenType.IDENTIFIER)
for t in code_tokens[3:])):
alias, symbol = code_tokens[1], code_tokens[3]
# Mark both tokens as an alias definition to avoid counting them as usages.
alias.metadata.is_alias_definition = True
symbol.metadata.is_alias_definition = True
return alias.string, tokenutil.GetIdentifierForToken(symbol)
def _AllFunctionPropertyAssignTokens(self, start_token, end_token):
"""Checks if tokens are (likely) a valid function property assignment.
Args:
start_token: Start of the token range.
end_token: End of the token range.
Returns:
True if all tokens between start_token and end_token are legal tokens
within a function declaration and assignment into a property.
"""
for token in tokenutil.GetTokenRange(start_token, end_token):
fn_decl_tokens = (Type.FUNCTION_DECLARATION,
Type.PARAMETERS,
Type.START_PARAMETERS,
Type.END_PARAMETERS,
Type.END_PAREN)
if (token.type not in fn_decl_tokens and
token.IsCode() and
not tokenutil.IsIdentifierOrDot(token) and
not token.IsAssignment() and
not (token.type == Type.OPERATOR and token.string == ',')):
return False
return True
