def make_identifier(self) -> Token:
id_str = ""
pos_start = self.pos.copy()
while self.current_char and (self.current_char.isalnum()
or self.current_char == "_"):
id_str += self.current_char
self.advance()
tok_type = TT_KEYWORD if id_str in KEYWORDS.values() else TT_IDENTIFIER
return StringToken(tok_type, pos_start, self.pos, id_str)
def t_ID(t):
r'[a-zA-Z][a-zA-Z0-9]*[$%]?'
t.type = reserved.get(t.value.lower(), 'ID')
if t.type != 'ID':
t.value = t.type
if t.type == 'BIN':
t.lexer.begin('bin')
return None
return t
def t_ID(t):
r'[a-zA-Z][a-zA-Z0-9]*[$%]?'
t.type = reserved.get(t.value.lower(), 'ID')
if t.type != 'ID':
t.value = t.type
if t.type == 'BIN':
t.lexer.begin('bin')
return None
return t
def t_ID(t):
r'[a-zA-Z][a-zA-Z0-9]*[$%]?'
t.type = reserved.get(t.value.lower(), 'ID')
callables = {
api.constants.CLASS.array: 'ARRAY_ID',
}
if t.type != 'ID':
t.value = t.type
else:
entry = api.global_.SYMBOL_TABLE.get_entry(
t.value) if api.global_.SYMBOL_TABLE is not None else None
if entry:
t.type = callables.get(entry.class_, t.type)
if t.type == 'BIN':
t.lexer.begin('bin')
return None
return t
'https://www.wsj.com/articles/uber-prepares-for-ipo-at-midpoint-of-target-range-or-lower-11557422774?mod=hp_lead_pos1',
'https://abcnews.go.com/US/guard-colorado-school-shooting-shot-deputies-wounded-student/story?id=62932261&cid=clicksource_4380645_null_hero_hed',
'https://www.bbc.com/news/business-48217766',
'https://www.usatoday.com/story/life/tv/2019/05/09/alex-trebek-pancreatic-cancer-cbs-sunday-morning-interview/1155575001/',
'https://www.latimes.com/local/lanow/la-me-ln-saenz-arrest-gun-investigation-20190509-story.html',
'https://www.foxnews.com/politics/two-years-after-getting-fired-by-trump-comeys-reign-at-fbi-remains-under-fire-amid-spygate',
'https://www.aol.com/article/news/2019/05/09/two-students-arrested-in-school-shooting-appear-in-court/23723544/',
'https://www.thedailybeast.com/stem-school-shooting-students-walk-out-of-vigil-organized-by-team-enough-and-moms-demand-action',
'https://www.avclub.com/why-didn-t-daenerys-torch-euron-who-s-side-is-jaime-on-1834599395',
'https://electrek.co/2019/05/09/tesla-saved-life-owner-crash-autopilot/',
'https://io9.gizmodo.com/one-of-avengers-endgames-most-incredible-scenes-is-now-1834644122',
'https://www.cbsnews.com/news/alabama-abortion-law-2019-chaos-on-senate-floor-today-while-state-lawmakers-debate-near-total-abortion-ban-2019-05-09/',
'https://techcrunch.com/2019/05/09/blue-origin-unveils-its-lunar-lander-blue-moon/',
'https://abc13.com/finance/walmart-managers-earn-$175000-a-year-on-average-report/5293048/',
'https://www.theverge.com/2019/5/9/18563474/google-pixel-3a-xl-teardown-easy-repairs',
'https://kotaku.com/the-final-fantasy-vii-remake-finally-re-emerges-1834654881',
'https://www.polygon.com/2019/5/9/18543880/ghost-recon-breakpoint-release-date-announcement-livestream-pc-ps4-xbox-one'
]
# times = []
# for _ in range(10):
#start = time.time()
for url in urls:
try:
article = KEYWORDS(url)
except:
print('dumbass this is a bad url')
k = article.get_keywords()
for word in k:
print(word)
preprocessor = {
'line': '_LINE',
'init': '_INIT',
'require': '_REQUIRE',
'pragma': '_PRAGMA',
'push': '_PUSH',
'pop': '_POP',
}
macros = (
'__LINE__',
'__FILE__',
)
tokens = _tokens + tuple(set(reserved.values())) + tuple(
preprocessor.values()) + macros
def t_INITIAL_bin_comment_beginBlockComment(t):
r"/'"
global __COMMENT_LEVEL
__COMMENT_LEVEL += 1
t.lexer.push_state('comment')
def t_comment_endBlockComment(t):
r"'/"
global __COMMENT_LEVEL
preprocessor = {
'line' : '_LINE',
'init' : '_INIT',
'require' : '_REQUIRE',
'pragma' : '_PRAGMA',
'push' : '_PUSH',
'pop' : '_POP',
}
macros = (
'__LINE__', '__FILE__',
)
tokens = _tokens + tuple(set(reserved.values())) + tuple(preprocessor.values()) + macros
def t_INITIAL_bin_comment_beginBlockComment(t):
r"/'"
global __COMMENT_LEVEL
__COMMENT_LEVEL += 1
t.lexer.push_state('comment')
def t_comment_endBlockComment(t):
r"'/"
global __COMMENT_LEVEL
__COMMENT_LEVEL -= 1
preprocessor = {
'line': '_LINE',
'init': '_INIT',
'require': '_REQUIRE',
'pragma': '_PRAGMA',
'push': '_PUSH',
'pop': '_POP',
}
macros = (
'__LINE__',
'__FILE__',
)
tokens = sorted(_tokens + tuple(set(reserved.values())) +
tuple(preprocessor.values()) + macros)
def t_INITIAL_bin_comment_beginBlockComment(t):
r"/'"
global __COMMENT_LEVEL
__COMMENT_LEVEL += 1
t.lexer.push_state('comment')
def t_comment_endBlockComment(t):
r"'/"
global __COMMENT_LEVEL
