def _initialize(self, dt): cl = self.txtinput_command_line self.txtinput_history_box.lexer = BashSessionLexer() self.txtinput_history_box.text = u''.join(self.textcache) self.txtinput_command_line.text = self.prompt() self.txtinput_command_line.bind(focus=self.on_focus) Clock.schedule_once(self._change_txtcache) self._focus(self.txtinput_command_line)
def _initialize(self, dt): '''Set console default variable values ''' cl = self.txtinput_command_line self.txtinput_history_box.lexer = BashSessionLexer() self.txtinput_history_box.text = u''.join(self.textcache) self.txtinput_command_line.text = self.prompt() self.txtinput_command_line.bind(focus=self.on_focus) self.txtinput_command_line.bind( selection_text=self.on_txtinput_selection) Clock.schedule_once(self._change_txtcache) self._focus(self.txtinput_command_line) self._list = [self.txtinput_command_line]
class BashSessionTest(unittest.TestCase): def setUp(self): self.lexer = BashSessionLexer() self.maxDiff = None def testNeedsName(self): fragment = u'$ echo \\\nhi\nhi\n' tokens = [ (Token.Text, u''), (Token.Generic.Prompt, u'$'), (Token.Text, u' '), (Token.Name.Builtin, u'echo'), (Token.Text, u' '), (Token.Literal.String.Escape, u'\\\n'), (Token.Text, u'hi'), (Token.Text, u'\n'), (Token.Generic.Output, u'hi\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
class BashSessionTest(unittest.TestCase): def setUp(self): self.lexer = BashSessionLexer() self.maxDiff = None def testNeedsName(self): fragment = u'$ echo \\\nhi\nhi\n' tokens = [ (Token.Text, u''), (Token.Generic.Prompt, u'$'), (Token.Text, u' '), (Token.Name.Builtin, u'echo'), (Token.Text, u' '), (Token.Literal.String.Escape, u'\\\n'), (Token.Text, u'hi'), (Token.Text, u'\n'), (Token.Generic.Output, u'hi\n'), ] self.assertEqual(tokens, list(self.lexer.get_tokens(fragment)))
def lexer_session(): yield BashSessionLexer()
def setUp(self): self.lexer = BashSessionLexer() self.maxDiff = None
def get_tokens_unprocessed(self, text): lualexer = LuaLexer(**self.options) ymllexer = YamlLexer(**self.options) shslexer = BashSessionLexer(**self.options) curcode = '' insertions = [] code = False prompt_len = 0 curyml = '' yml = False ymlcnt = 0 curshs = '' shs = False for match in line_re.finditer(text): line = match.group() # print line # print code, yml, shs # First part - if output starts from '$ ' then it's BASH session # - We must only check that we're not inside of YAML # code can't start with '$ ' # if output (not inside YAML) starts with '$ ' - it's not our problem # Also, we can match multiline commands only if line ends with '\' check_shs = (line.startswith('$ ') and not yml) or shs if check_shs: curshs += line if line.endswith('\\'): shs = True continue for item in shslexer.get_tokens_unprocessed(curshs): yield item curshs = '' shs = False continue # Second part - check for YAML # 1) It's begin, means (yml == False) and line.strip() == '---' # 2) It's middle. (yml == True) and line.strip() not in ('---', '...') # 3) It's end - then (yml == False) and line.strip() == '...'] check_yml_begin = (yml == False and line.strip() in (yml_beg, )) # check_yml_middle = (yml == True and line.strip() not in (yml_beg, yml_end)) check_yml_end = (yml == True and line.strip() == yml_end and ymlcnt == 0) if (check_yml_begin or yml): # print check_yml_begin, check_yml_middle, check_yml_end # Flush previous code buffers if (yml is True and line.strip() == yml_beg): ymlcnt += 1 if (not check_yml_end and line.strip() == yml_end): ymlcnt += 1 if check_yml_begin and curcode: for item in do_insertions( insertions, lualexer.get_tokens_unprocessed(curcode)): yield item code = False curcode = '' insertions = [] curyml += line # We finished reading YAML output, so push it to user if check_yml_end: for item in ymllexer.get_tokens_unprocessed(curyml): yield item curyml = '' yml = False if check_yml_end else True # print 'yaml gotcha %d' % yml continue # Third part - check for Tarantool's Lua # It's combination of: # prompt: tarantool> or localhost> or localhost:{port}> # lua: body after prompt + space prompt_pos_flexible = find_prompt(line) prompt_pos_strict = prompt_pos_flexible if not code else None if prompt_pos_strict: prompt_len = prompt_pos_strict + 2 check_code_begin = bool(prompt_pos_strict) check_code_middle = code and line.startswith(' ' * (prompt_len - 2) + '> ') check_code_flexible = False # e.g. we have two 'tarantool> ' in a row - code is True and # check_code_middle is False then we have to do something about it, # otherwise it will be like Generic.Output if code and check_code_middle is False and bool( prompt_pos_flexible): prompt_len = prompt_pos_flexible + 2 check_code_flexible = True if (check_code_begin or check_code_middle or check_code_flexible): code = True insertions.append( (len(curcode), [(0, Generic.Prompt, line[:prompt_len])])) curcode += line[prompt_len:] continue # If it's not something before - then we must check for code # and push that line as 'Generic.Output' if curcode: for item in do_insertions( insertions, lualexer.get_tokens_unprocessed(curcode)): yield item code = False curcode = '' insertions = [] yield match.start(), Generic.Output, line if curcode: for item in do_insertions( insertions, lualexer.get_tokens_unprocessed(curcode)): yield item if curyml: for item in ymllexer.get_tokens_unprocessed(curyml): yield item if curshs: for item in shslexer.get_tokens_unprocessed(curshs): yield item
def setUp(self): self.lexer = BashSessionLexer() self.maxDiff = None
#!/usr/bin/env python import argparse from pygments.lexers import BashSessionLexer # ============================================================================= parser = argparse.ArgumentParser( description=('Prints out the tokens ' 'generated by pygments.lexers.BashSessionLexer')) parser.add_argument('files', type=str, nargs='+', help='One or more file names to lex and parse') args = parser.parse_args() # --- Do the parsing lexer = BashSessionLexer() with open(args.files[0]) as f: contents = f.read() for token, text in lexer.get_tokens(contents): if text == '\n': text = '\\n' print('%s: %s' % (token, text))
def get_tokens_unprocessed(self, text): lualexer = LuaLexer(**self.options) ymllexer = YamlLexer(**self.options) shslexer = BashSessionLexer(**self.options) curcode = '' insertions = [] code = False prompt_len = 0 curyml = '' yml = False ymlcnt = 0 curshs = '' shs = False for match in line_re.finditer(text): line = match.group() # print line # print code, yml, shs # First part - if output starts from '$ ' then it's BASH session # - We must only check that we're not inside of YAML # code can't start with '$ ' # if output (not inside YAML) starts with '$ ' - it's not our problem # Also, we can match multiline commands only if line ends with '\' check_shs = (line.startswith('$ ') and not yml) or shs if check_shs: curshs += line if line.endswith('\\'): shs = True continue for item in shslexer.get_tokens_unprocessed(curshs): yield item curshs = '' shs = False continue # Second part - check for YAML # 1) It's begin, means (yml == False) and line.strip() == '---' # 2) It's middle. (yml == True) and line.strip() not in ('---', '...') # 3) It's end - then (yml == False) and line.strip() == '...'] check_yml_begin = (yml == False and line.strip() in (yml_beg, )) # check_yml_middle = (yml == True and line.strip() not in (yml_beg, yml_end)) check_yml_end = (yml == True and line.strip() == yml_end and ymlcnt == 0) if (check_yml_begin or yml): # print check_yml_begin, check_yml_middle, check_yml_end # Flush previous code buffers if (yml is True and line.strip() == yml_beg): ymlcnt += 1 if (not check_yml_end and line.strip() == yml_end): ymlcnt += 1 if check_yml_begin and curcode: for item in do_insertions(insertions, lualexer.get_tokens_unprocessed(curcode)): yield item code = False curcode = '' insertions = [] curyml += line # We finished reading YAML output, so push it to user if check_yml_end: for item in ymllexer.get_tokens_unprocessed(curyml): yield item curyml = '' yml = False if check_yml_end else True # print 'yaml gotcha %d' % yml continue # Third part - check for Tarantool's Lua # It's combination of: # prompt: tarantool> or localhost> or localhost:{port}> # lua: body after prompt + space prompt_pos_flexible = find_prompt(line) prompt_pos_strict = prompt_pos_flexible if not code else None if prompt_pos_strict: prompt_len = prompt_pos_strict + 2 check_code_begin = bool(prompt_pos_strict) check_code_middle = code and line.startswith(' ' * (prompt_len - 2) + '> ') check_code_flexible = False # e.g. we have two 'tarantool> ' in a row - code is True and # check_code_middle is False then we have to do something about it, # otherwise it will be like Generic.Output if code and check_code_middle is False and bool(prompt_pos_flexible): prompt_len = prompt_pos_flexible + 2 check_code_flexible = True if (check_code_begin or check_code_middle or check_code_flexible): code = True insertions.append((len(curcode), [(0, Generic.Prompt, line[:prompt_len])])) curcode += line[prompt_len:] continue # If it's not something before - then we must check for code # and push that line as 'Generic.Output' if curcode: for item in do_insertions(insertions, lualexer.get_tokens_unprocessed(curcode)): yield item code = False curcode = '' insertions = [] yield match.start(), Generic.Output, line if curcode: for item in do_insertions(insertions, lualexer.get_tokens_unprocessed(curcode)): yield item if curyml: for item in ymllexer.get_tokens_unprocessed(curyml): yield item if curshs: for item in shslexer.get_tokens_unprocessed(curshs): yield item