def testGetWholeIdentifierString(self):
"""Tests that created identifiers satisfy usage of the identifier."""
input_lines = ['package.Foo.', ' veryLong.', ' identifier;']
token = testutil.TokenizeSource(input_lines)
self.assertEquals('package.Foo.veryLong.identifier',
tokenutil.GetIdentifierForToken(token))
self.assertEquals(None, tokenutil.GetIdentifierForToken(token.next))
def GetTargetIdentifier(self):
"""Returns the identifier (as a string) that this is a comment for.
Note that this uses method uses GetIdentifierForToken to get the full
identifier, even if broken up by whitespace, newlines, or comments,
and thus could be longer than GetTargetToken().string.
Returns:
The identifier for the token this comment is for.
"""
token = self.GetTargetToken()
if token:
return tokenutil.GetIdentifierForToken(token)
def MatchAlias(context):
"""Match an alias statement (some identifier assigned to a variable).
Example alias: var MyClass = proj.longNamespace.MyClass.
Args:
context: An EcmaContext of type EcmaContext.VAR.
Returns:
If a valid alias, returns a tuple of alias and symbol, otherwise None.
"""
if context.type != ecmametadatapass.EcmaContext.VAR:
return
# The var's parent is a STATEMENT, which should be directly below goog.scope.
if not IsGoogScopeBlock(context.parent.parent):
return
# Get the tokens in this statement.
if context.start_token and context.end_token:
statement_tokens = tokenutil.GetTokenRange(context.start_token,
context.end_token)
else:
return
# And now just those tokens that are actually code.
is_non_code_type = lambda t: t.type not in JavaScriptTokenType.NON_CODE_TYPES
code_tokens = filter(is_non_code_type, statement_tokens)
# This section identifies statements of the alias form "var alias = symbol".
# Pop off the semicolon if present.
if code_tokens and code_tokens[-1].IsType(JavaScriptTokenType.SEMICOLON):
code_tokens.pop()
if len(code_tokens) < 4:
return
# Verify that this is of the form "var lvalue = identifier;".
# The identifier may span multiple lines and could be multiple tokens.
if (code_tokens[0].IsKeyword('var')
and code_tokens[1].IsType(JavaScriptTokenType.SIMPLE_LVALUE)
and code_tokens[2].IsOperator('=') and all(
t.IsType(JavaScriptTokenType.IDENTIFIER)
for t in code_tokens[3:])):
alias, symbol = code_tokens[1], code_tokens[3]
# Mark both tokens as an alias definition to avoid counting them as usages.
alias.metadata.is_alias_definition = True
symbol.metadata.is_alias_definition = True
return alias.string, tokenutil.GetIdentifierForToken(symbol)
def _ProcessBlock(self, context, global_alias_map):
"""Scans a goog.scope block to find aliases and mark alias tokens."""
alias_map = global_alias_map.copy()
# Iterate over every token in the context. Each token points to one
# context, but multiple tokens may point to the same context. We only want
# to check each context once, so keep track of those we've seen.
seen_contexts = set()
token = context.start_token
while token and self._IsTokenInParentBlock(token, context):
token_context = token.metadata.context if token.metadata else None
# Check to see if this token is an alias.
if token_context and token_context not in seen_contexts:
seen_contexts.add(token_context)
# If this is a alias statement in the goog.scope block.
if (token_context.type == ecmametadatapass.EcmaContext.VAR
and scopeutil.IsGoogScopeBlock(
token_context.parent.parent)):
match = scopeutil.MatchAlias(token_context)
# If this is an alias, remember it in the map.
if match:
alias, symbol = match
symbol = _GetAliasForIdentifier(symbol,
alias_map) or symbol
if scopeutil.IsInClosurizedNamespace(
symbol, self._closurized_namespaces):
alias_map[alias] = symbol
# If this token is an identifier that matches an alias,
# mark the token as an alias to the original symbol.
if (token.type is
javascripttokens.JavaScriptTokenType.SIMPLE_LVALUE
or token.type is
javascripttokens.JavaScriptTokenType.IDENTIFIER):
identifier = tokenutil.GetIdentifierForToken(token)
if identifier:
aliased_symbol = _GetAliasForIdentifier(
identifier, alias_map)
if aliased_symbol:
token.metadata.aliased_symbol = aliased_symbol
elif token.type == javascripttokens.JavaScriptTokenType.DOC_FLAG:
flag = token.attached_object
if flag and flag.HasType() and flag.jstype:
_SetTypeAlias(flag.jstype, alias_map)
token = token.next # Get next token
def MatchAlias(context):
"""Match an alias statement (some identifier assigned to a variable).
Example alias: var MyClass = proj.longNamespace.MyClass.
Args:
context: An EcmaContext of type EcmaContext.VAR.
Returns:
If a valid alias, returns a tuple of alias and symbol, otherwise None.
"""
code_tokens = _GetVarAssignmentTokens(context)
if code_tokens is None:
return
if all(tokenutil.IsIdentifierOrDot(t) for t in code_tokens[3:]):
# var Foo = bar.Foo;
alias, symbol = code_tokens[1], code_tokens[3]
# Mark both tokens as an alias definition to not count them as usages.
alias.metadata.is_alias_definition = True
symbol.metadata.is_alias_definition = True
return alias.string, tokenutil.GetIdentifierForToken(symbol)
def ProcessToken(self, token, state_tracker):
"""Processes the given token for dependency information.
Args:
token: The token to process.
state_tracker: The JavaScript state tracker.
"""
# Note that this method is in the critical path for the linter and has been
# optimized for performance in the following ways:
# - Tokens are checked by type first to minimize the number of function
# calls necessary to determine if action needs to be taken for the token.
# - The most common tokens types are checked for first.
# - The number of function calls has been minimized (thus the length of this
# function.
if token.type == TokenType.IDENTIFIER:
# TODO(user): Consider saving the whole identifier in metadata.
whole_identifier_string = tokenutil.GetIdentifierForToken(token)
if whole_identifier_string is None:
# We only want to process the identifier one time. If the whole string
# identifier is None, that means this token was part of a multi-token
# identifier, but it was not the first token of the identifier.
return
# In the odd case that a goog.require is encountered inside a function,
# just ignore it (e.g. dynamic loading in test runners).
if token.string == 'goog.require' and not state_tracker.InFunction():
self._require_tokens.append(token)
namespace = tokenutil.GetStringAfterToken(token)
if namespace in self._required_namespaces:
self._duplicate_require_tokens.append(token)
else:
self._required_namespaces.append(namespace)
# If there is a suppression for the require, add a usage for it so it
# gets treated as a regular goog.require (i.e. still gets sorted).
jsdoc = state_tracker.GetDocComment()
if jsdoc and ('extraRequire' in jsdoc.suppressions):
self._suppressed_requires.append(namespace)
self._AddUsedNamespace(state_tracker, namespace, token.line_number)
elif token.string == 'goog.provide':
self._provide_tokens.append(token)
namespace = tokenutil.GetStringAfterToken(token)
if namespace in self._provided_namespaces:
self._duplicate_provide_tokens.append(token)
else:
self._provided_namespaces.append(namespace)
# If there is a suppression for the provide, add a creation for it so it
# gets treated as a regular goog.provide (i.e. still gets sorted).
jsdoc = state_tracker.GetDocComment()
if jsdoc and ('extraProvide' in jsdoc.suppressions):
self._AddCreatedNamespace(state_tracker, namespace, token.line_number)
elif token.string == 'goog.scope':
self._scopified_file = True
elif token.string == 'goog.setTestOnly':
# Since the message is optional, we don't want to scan to later lines.
for t in tokenutil.GetAllTokensInSameLine(token):
if t.type == TokenType.STRING_TEXT:
message = t.string
if re.match(r'^\w+(\.\w+)+$', message):
# This looks like a namespace. If it's a Closurized namespace,
# consider it created.
base_namespace = message.split('.', 1)[0]
if base_namespace in self._closurized_namespaces:
self._AddCreatedNamespace(state_tracker, message,
token.line_number)
break
else:
jsdoc = state_tracker.GetDocComment()
if token.metadata and token.metadata.aliased_symbol:
whole_identifier_string = token.metadata.aliased_symbol
if jsdoc and jsdoc.HasFlag('typedef'):
self._AddCreatedNamespace(state_tracker, whole_identifier_string,
token.line_number,
namespace=self.GetClosurizedNamespace(
whole_identifier_string))
else:
if not (token.metadata and token.metadata.is_alias_definition):
self._AddUsedNamespace(state_tracker, whole_identifier_string,
token.line_number)
elif token.type == TokenType.SIMPLE_LVALUE:
identifier = token.values['identifier']
start_token = tokenutil.GetIdentifierStart(token)
if start_token and start_token != token:
# Multi-line identifier being assigned. Get the whole identifier.
identifier = tokenutil.GetIdentifierForToken(start_token)
else:
start_token = token
# If an alias is defined on the start_token, use it instead.
if (start_token and
start_token.metadata and
start_token.metadata.aliased_symbol and
not start_token.metadata.is_alias_definition):
identifier = start_token.metadata.aliased_symbol
if identifier:
namespace = self.GetClosurizedNamespace(identifier)
if state_tracker.InFunction():
self._AddUsedNamespace(state_tracker, identifier, token.line_number)
elif namespace and namespace != 'goog':
self._AddCreatedNamespace(state_tracker, identifier,
token.line_number, namespace=namespace)
elif token.type == TokenType.DOC_FLAG:
flag_type = token.attached_object.flag_type
is_interface = state_tracker.GetDocComment().HasFlag('interface')
if flag_type == 'implements' or (flag_type == 'extends' and is_interface):
# Interfaces should be goog.require'd.
doc_start = tokenutil.Search(token, TokenType.DOC_START_BRACE)
interface = tokenutil.Search(doc_start, TokenType.COMMENT)
self._AddUsedNamespace(state_tracker, interface.string,
token.line_number)
def testGetIdentifierForToken(self):
tokens = testutil.TokenizeSource("""
start1.abc.def.prototype.
onContinuedLine
(start2.abc.def
.hij.klm
.nop)
start3.abc.def
.hij = function() {};
// An absurd multi-liner.
start4.abc.def.
hij.
klm = function() {};
start5 . aaa . bbb . ccc
shouldntBePartOfThePreviousSymbol
start6.abc.def ghi.shouldntBePartOfThePreviousSymbol
var start7 = 42;
function start8() {
}
start9.abc. // why is there a comment here?
def /* another comment */
shouldntBePart
start10.abc // why is there a comment here?
.def /* another comment */
shouldntBePart
start11.abc. middle1.shouldNotBeIdentifier
""")
def _GetTokenStartingWith(token_starts_with):
for t in tokens:
if t.string.startswith(token_starts_with):
return t
self.assertEquals(
'start1.abc.def.prototype.onContinuedLine',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start1')))
self.assertEquals(
'start2.abc.def.hij.klm.nop',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start2')))
self.assertEquals(
'start3.abc.def.hij',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start3')))
self.assertEquals(
'start4.abc.def.hij.klm',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start4')))
self.assertEquals(
'start5.aaa.bbb.ccc',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start5')))
self.assertEquals(
'start6.abc.def',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start6')))
self.assertEquals(
'start7',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start7')))
self.assertEquals(
'start8',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start8')))
self.assertEquals(
'start9.abc.def',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start9')))
self.assertEquals(
'start10.abc.def',
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('start10')))
self.assertIsNone(
tokenutil.GetIdentifierForToken(_GetTokenStartingWith('middle1')))
