def get_pygments_and_allow_to_wrap(text, pygmenttextlexer, ssym_dt):
pygmentstext = highlight(
mypatmat.ensure_thissym_around_syms(text, ssym_dt, " "), pygmenttextlexer, HtmlFormatter(noclasses=False)
)
(replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
pygmentstext, "<pre.*?>", replacewith='<span class="highlight">'
)
(replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
pygmentstext, "</pre>", replacewith="</span>"
)
(replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
pygmentstext, "<div.*?>", replacewith='<span class="highlight">'
)
(replaced, pygmentstext) = mypatmat.remove_all_in_between_on_same_line2_UNICODE(
pygmentstext, "</div>", replacewith="</span>"
)
return pygmentstext
def get_data_str(source,keys_dt,ssym_dt,STRINGS_REGEX,DIGITS_REGEX,TYPES_REGEX):
tokens = []
variables = []
SPACE = mypatmat.SPACES[0]
(strings_removed,source) = mypatmat.remove_all_in_between_on_same_line2(source,STRINGS_REGEX,replacewith=' '+STRING_PLACEHOLDER+' ')
types_removed = []
(digits_removed,source) = mypatmat.remove_all_in_between_on_same_line2(source,DIGITS_REGEX,replacewith=' '+DIGITS_PLACEHOLDER+' ')
source = mypatmat.replace_whitespace_with_single_space(source)
source = mypatmat.ensure_thissym_around_syms(source,ssym_dt,SPACE)
token_lines = source.split(SPACE)
pos_in_source = 0
num = 0
strings_num = 0
digits_num = 0
variables_num = 0
types_num = 0
for tok in token_lines:
# symbol token
if tok in ssym_dt:
tokens.append((tok,pos_in_source))
# keyword token
elif tok in keys_dt:
tokens.append((tok,pos_in_source,KEYWORD_CONSTANT))
elif tok != None and len(tok)>0:
if len(tok)==1:
if ord(tok)==ord(DIGITS_PLACEHOLDER):
tokens.append((DIGITS_PLACEHOLDER,pos_in_source,strings_num))
strings_num += 1
elif ord(tok)==ord(STRING_PLACEHOLDER):
tokens.append((STRING_PLACEHOLDER,pos_in_source,digits_num))
digits_num += 1
elif ord(tok)==ord(TYPES_PLACEHOLDER):
tokens.append((TYPES_PLACEHOLDER,pos_in_source,types_num))
types_num += 1
else: # variables (identifier)
tokens.append((VARIABLE_PLACEHOLDER,pos_in_source,num))
variables.append(tok)
variables_num += 1
num += 1
elif tok!=SPACE: # variable (identifier)
tokens.append((VARIABLE_PLACEHOLDER,pos_in_source,num))
num += 1
variables.append(tok)
variables_num += 1
pos_in_source += len(tok) + 1 # + 1 for the SPACE
tokens_str = tokens_to_str(tokens)
return (source,tokens,tokens_str,variables,strings_removed,digits_removed,types_removed)
