def generate_tokens(source_code, _=None): regx_regx = r"|/(?:\\.|[^/])+?/[igm]*" regx_pattern = re.compile(regx_regx) word_pattern = re.compile(r'\w+') tokens = CodeReader.generate_tokens(source_code, regx_regx) leading_by_word = False for token in tokens: if leading_by_word and regx_pattern.match(token): for subtoken in CodeReader.generate_tokens(token): yield subtoken else: yield token if not token.isspace(): leading_by_word = word_pattern.match(token)
def generate_tokens(source_code, _=None): return CodeReader.generate_tokens(source_code, r"|\'\'\'.*?\'\'\'" + r"|\"\"\".*?\"\"\"")
def generate_tokens(source_code, _=None): return CodeReader.generate_tokens( source_code, r'|' + r'|'.join(re.escape(s) for s in ( '..', '->', '<@', '@>', '@lazy', '@fuzzy', '@index', '@deterministic')))
def test_unknown_extension(self): self.assertEqual(None, CodeReader.get_reader("a.unknown"));
def test_not_case_sensitive(self): self.assertEqual(CLikeReader, CodeReader.get_reader("a.Cpp"))
def test_JavaScript(self): self.assertEqual(JavaScriptReader, CodeReader.get_reader("a.js"))
def test_c_cpp(self): for name in ("a.cpp", ".cxx", ".h", ".hpp"): self.assertEqual(CLikeReader, CodeReader.get_reader(name), "File name '%s' is not recognized as c/c++ file" % name);
def test_objectiveC(self): self.assertEqual(ObjCReader, CodeReader.get_reader("a.m"))
def test_java(self): self.assertEqual(JavaReader, CodeReader.get_reader("a.java"))
def test_new_reader_should_be_found(self): class NewReader(CodeReader): ext = ['ext'] self.assertEqual(NewReader, CodeReader.get_reader("a.ext")); del NewReader