def generate_tokens(source_code, _=None):
regx_regx = r"|/(?:\\.|[^/])+?/[igm]*"
regx_pattern = re.compile(regx_regx)
word_pattern = re.compile(r'\w+')
tokens = CodeReader.generate_tokens(source_code, regx_regx)
leading_by_word = False
for token in tokens:
if leading_by_word and regx_pattern.match(token):
for subtoken in CodeReader.generate_tokens(token):
yield subtoken
else:
yield token
if not token.isspace():
leading_by_word = word_pattern.match(token)
def generate_tokens(source_code, _=None):
return CodeReader.generate_tokens(source_code, r"|\'\'\'.*?\'\'\'" + r"|\"\"\".*?\"\"\"")
def generate_tokens(source_code, _=None):
return CodeReader.generate_tokens(
source_code,
r'|' + r'|'.join(re.escape(s) for s in (
'..', '->', '<@', '@>', '@lazy', '@fuzzy',
'@index', '@deterministic')))
def test_unknown_extension(self):
self.assertEqual(None, CodeReader.get_reader("a.unknown"));
def test_not_case_sensitive(self):
self.assertEqual(CLikeReader, CodeReader.get_reader("a.Cpp"))
def test_JavaScript(self):
self.assertEqual(JavaScriptReader, CodeReader.get_reader("a.js"))
def test_c_cpp(self):
for name in ("a.cpp", ".cxx", ".h", ".hpp"):
self.assertEqual(CLikeReader, CodeReader.get_reader(name),
"File name '%s' is not recognized as c/c++ file" % name);
def test_objectiveC(self):
self.assertEqual(ObjCReader, CodeReader.get_reader("a.m"))
def test_java(self):
self.assertEqual(JavaReader, CodeReader.get_reader("a.java"))
def test_new_reader_should_be_found(self):
class NewReader(CodeReader):
ext = ['ext']
self.assertEqual(NewReader, CodeReader.get_reader("a.ext"));
del NewReader
