コード例 #1
0
ファイル: dupfuns.py プロジェクト: rabeal/procode
def get_pygments_and_allow_to_wrap(text, pygmenttextlexer, ssym_dt):
    pygmentstext = highlight(
        mypatmat.ensure_thissym_around_syms(text, ssym_dt, " "), pygmenttextlexer, HtmlFormatter(noclasses=False)
    )
    (replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
        pygmentstext, "<pre.*?>", replacewith='<span class="highlight">'
    )
    (replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
        pygmentstext, "</pre>", replacewith="</span>"
    )
    (replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
        pygmentstext, "<div.*?>", replacewith='<span class="highlight">'
    )
    (replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
        pygmentstext, "</div>", replacewith="</span>"
    )
    return pygmentstext
コード例 #2
0
def get_data_str(source,keys_dt,ssym_dt,STRINGS_REGEX,DIGITS_REGEX,TYPES_REGEX):
  tokens = []
  variables = []
  SPACE = mypatmat.SPACES[0]
  

  (strings_removed,source) = mypatmat.remove_all_in_between_on_same_line2(source,STRINGS_REGEX,replacewith=' '+STRING_PLACEHOLDER+' ')

  types_removed = []

  (digits_removed,source) = mypatmat.remove_all_in_between_on_same_line2(source,DIGITS_REGEX,replacewith=' '+DIGITS_PLACEHOLDER+' ')

  source = mypatmat.replace_whitespace_with_single_space(source)
  source = mypatmat.ensure_thissym_around_syms(source,ssym_dt,SPACE)
  
  token_lines = source.split(SPACE)
  

  pos_in_source = 0
  num = 0
  strings_num = 0
  digits_num = 0
  variables_num = 0
  types_num = 0
  for tok in token_lines:
    # symbol token
    if tok in ssym_dt:
      tokens.append((tok,pos_in_source))
    # keyword token
    elif tok in keys_dt:
      tokens.append((tok,pos_in_source,KEYWORD_CONSTANT))
    
    elif tok != None and len(tok)>0:
      if len(tok)==1:
        if ord(tok)==ord(DIGITS_PLACEHOLDER):
          tokens.append((DIGITS_PLACEHOLDER,pos_in_source,strings_num))
          strings_num += 1
        elif ord(tok)==ord(STRING_PLACEHOLDER):
          tokens.append((STRING_PLACEHOLDER,pos_in_source,digits_num))
          digits_num += 1
        elif ord(tok)==ord(TYPES_PLACEHOLDER):
          tokens.append((TYPES_PLACEHOLDER,pos_in_source,types_num))
          types_num += 1
        else: # variables (identifier)
          tokens.append((VARIABLE_PLACEHOLDER,pos_in_source,num))
          variables.append(tok)
          variables_num += 1
        num += 1
      elif tok!=SPACE:  # variable (identifier)
        tokens.append((VARIABLE_PLACEHOLDER,pos_in_source,num))
        num += 1
        variables.append(tok)
        variables_num += 1
    pos_in_source += len(tok) + 1 # + 1 for the SPACE

  tokens_str = tokens_to_str(tokens)

  


  return (source,tokens,tokens_str,variables,strings_removed,digits_removed,types_removed)