def get_tokens_unprocessed(self, text): lualexer = LuaLexer(**self.options) ymllexer = YamlLexer(**self.options) shslexer = BashSessionLexer(**self.options) curcode = '' insertions = [] code = False prompt_len = 0 curyml = '' yml = False ymlcnt = 0 curshs = '' shs = False for match in line_re.finditer(text): line = match.group() # print line # print code, yml, shs # First part - if output starts from '$ ' then it's BASH session # - We must only check that we're not inside of YAML # code can't start with '$ ' # if output (not inside YAML) starts with '$ ' - it's not our problem # Also, we can match multiline commands only if line ends with '\' check_shs = (line.startswith('$ ') and not yml) or shs if check_shs: curshs += line if line.endswith('\\'): shs = True continue for item in shslexer.get_tokens_unprocessed(curshs): yield item curshs = '' shs = False continue # Second part - check for YAML # 1) It's begin, means (yml == False) and line.strip() == '---' # 2) It's middle. (yml == True) and line.strip() not in ('---', '...') # 3) It's end - then (yml == False) and line.strip() == '...'] check_yml_begin = (yml == False and line.strip() in (yml_beg, )) # check_yml_middle = (yml == True and line.strip() not in (yml_beg, yml_end)) check_yml_end = (yml == True and line.strip() == yml_end and ymlcnt == 0) if (check_yml_begin or yml): # print check_yml_begin, check_yml_middle, check_yml_end # Flush previous code buffers if (yml is True and line.strip() == yml_beg): ymlcnt += 1 if (not check_yml_end and line.strip() == yml_end): ymlcnt += 1 if check_yml_begin and curcode: for item in do_insertions(insertions, lualexer.get_tokens_unprocessed(curcode)): yield item code = False curcode = '' insertions = [] curyml += line # We finished reading YAML output, so push it to user if check_yml_end: for item in ymllexer.get_tokens_unprocessed(curyml): yield item curyml = '' yml = False if check_yml_end else True # print 'yaml gotcha %d' % yml continue # Third part - check for Tarantool's Lua # It's combination of: # prompt: tarantool> or localhost> or localhost:{port}> # lua: body after prompt + space prompt_pos_flexible = find_prompt(line) prompt_pos_strict = prompt_pos_flexible if not code else None if prompt_pos_strict: prompt_len = prompt_pos_strict + 2 check_code_begin = bool(prompt_pos_strict) check_code_middle = code and line.startswith(' ' * (prompt_len - 2) + '> ') check_code_flexible = False # e.g. we have two 'tarantool> ' in a row - code is True and # check_code_middle is False then we have to do something about it, # otherwise it will be like Generic.Output if code and check_code_middle is False and bool(prompt_pos_flexible): prompt_len = prompt_pos_flexible + 2 check_code_flexible = True if (check_code_begin or check_code_middle or check_code_flexible): code = True insertions.append((len(curcode), [(0, Generic.Prompt, line[:prompt_len])])) curcode += line[prompt_len:] continue # If it's not something before - then we must check for code # and push that line as 'Generic.Output' if curcode: for item in do_insertions(insertions, lualexer.get_tokens_unprocessed(curcode)): yield item code = False curcode = '' insertions = [] yield match.start(), Generic.Output, line if curcode: for item in do_insertions(insertions, lualexer.get_tokens_unprocessed(curcode)): yield item if curyml: for item in ymllexer.get_tokens_unprocessed(curyml): yield item if curshs: for item in shslexer.get_tokens_unprocessed(curshs): yield item