def make_identifier(self) -> Token: id_str = "" pos_start = self.pos.copy() while self.current_char and (self.current_char.isalnum() or self.current_char == "_"): id_str += self.current_char self.advance() tok_type = TT_KEYWORD if id_str in KEYWORDS.values() else TT_IDENTIFIER return StringToken(tok_type, pos_start, self.pos, id_str)
def t_ID(t): r'[a-zA-Z][a-zA-Z0-9]*[$%]?' t.type = reserved.get(t.value.lower(), 'ID') if t.type != 'ID': t.value = t.type if t.type == 'BIN': t.lexer.begin('bin') return None return t
def t_ID(t): r'[a-zA-Z][a-zA-Z0-9]*[$%]?' t.type = reserved.get(t.value.lower(), 'ID') callables = { api.constants.CLASS.array: 'ARRAY_ID', } if t.type != 'ID': t.value = t.type else: entry = api.global_.SYMBOL_TABLE.get_entry( t.value) if api.global_.SYMBOL_TABLE is not None else None if entry: t.type = callables.get(entry.class_, t.type) if t.type == 'BIN': t.lexer.begin('bin') return None return t
'https://www.wsj.com/articles/uber-prepares-for-ipo-at-midpoint-of-target-range-or-lower-11557422774?mod=hp_lead_pos1', 'https://abcnews.go.com/US/guard-colorado-school-shooting-shot-deputies-wounded-student/story?id=62932261&cid=clicksource_4380645_null_hero_hed', 'https://www.bbc.com/news/business-48217766', 'https://www.usatoday.com/story/life/tv/2019/05/09/alex-trebek-pancreatic-cancer-cbs-sunday-morning-interview/1155575001/', 'https://www.latimes.com/local/lanow/la-me-ln-saenz-arrest-gun-investigation-20190509-story.html', 'https://www.foxnews.com/politics/two-years-after-getting-fired-by-trump-comeys-reign-at-fbi-remains-under-fire-amid-spygate', 'https://www.aol.com/article/news/2019/05/09/two-students-arrested-in-school-shooting-appear-in-court/23723544/', 'https://www.thedailybeast.com/stem-school-shooting-students-walk-out-of-vigil-organized-by-team-enough-and-moms-demand-action', 'https://www.avclub.com/why-didn-t-daenerys-torch-euron-who-s-side-is-jaime-on-1834599395', 'https://electrek.co/2019/05/09/tesla-saved-life-owner-crash-autopilot/', 'https://io9.gizmodo.com/one-of-avengers-endgames-most-incredible-scenes-is-now-1834644122', 'https://www.cbsnews.com/news/alabama-abortion-law-2019-chaos-on-senate-floor-today-while-state-lawmakers-debate-near-total-abortion-ban-2019-05-09/', 'https://techcrunch.com/2019/05/09/blue-origin-unveils-its-lunar-lander-blue-moon/', 'https://abc13.com/finance/walmart-managers-earn-$175000-a-year-on-average-report/5293048/', 'https://www.theverge.com/2019/5/9/18563474/google-pixel-3a-xl-teardown-easy-repairs', 'https://kotaku.com/the-final-fantasy-vii-remake-finally-re-emerges-1834654881', 'https://www.polygon.com/2019/5/9/18543880/ghost-recon-breakpoint-release-date-announcement-livestream-pc-ps4-xbox-one' ] # times = [] # for _ in range(10): #start = time.time() for url in urls: try: article = KEYWORDS(url) except: print('dumbass this is a bad url') k = article.get_keywords() for word in k: print(word)
preprocessor = { 'line': '_LINE', 'init': '_INIT', 'require': '_REQUIRE', 'pragma': '_PRAGMA', 'push': '_PUSH', 'pop': '_POP', } macros = ( '__LINE__', '__FILE__', ) tokens = _tokens + tuple(set(reserved.values())) + tuple( preprocessor.values()) + macros def t_INITIAL_bin_comment_beginBlockComment(t): r"/'" global __COMMENT_LEVEL __COMMENT_LEVEL += 1 t.lexer.push_state('comment') def t_comment_endBlockComment(t): r"'/" global __COMMENT_LEVEL
preprocessor = { 'line' : '_LINE', 'init' : '_INIT', 'require' : '_REQUIRE', 'pragma' : '_PRAGMA', 'push' : '_PUSH', 'pop' : '_POP', } macros = ( '__LINE__', '__FILE__', ) tokens = _tokens + tuple(set(reserved.values())) + tuple(preprocessor.values()) + macros def t_INITIAL_bin_comment_beginBlockComment(t): r"/'" global __COMMENT_LEVEL __COMMENT_LEVEL += 1 t.lexer.push_state('comment') def t_comment_endBlockComment(t): r"'/" global __COMMENT_LEVEL __COMMENT_LEVEL -= 1
preprocessor = { 'line': '_LINE', 'init': '_INIT', 'require': '_REQUIRE', 'pragma': '_PRAGMA', 'push': '_PUSH', 'pop': '_POP', } macros = ( '__LINE__', '__FILE__', ) tokens = sorted(_tokens + tuple(set(reserved.values())) + tuple(preprocessor.values()) + macros) def t_INITIAL_bin_comment_beginBlockComment(t): r"/'" global __COMMENT_LEVEL __COMMENT_LEVEL += 1 t.lexer.push_state('comment') def t_comment_endBlockComment(t): r"'/" global __COMMENT_LEVEL