def pixmap(cursor, num_lines=6, scale=0.8):
"""Return a QPixmap displaying the selected lines of the document.
If the cursor has no selection, num_lines are drawn.
By default the text is drawn 0.8 * the normal font size. You can change
that by supplying the scale parameter.
"""
block = cursor.document().findBlock(cursor.selectionStart())
c2 = QTextCursor(block)
if cursor.hasSelection():
c2.setPosition(cursor.selectionEnd(), QTextCursor.KeepAnchor)
c2.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
else:
c2.movePosition(QTextCursor.NextBlock, QTextCursor.KeepAnchor, num_lines)
data = textformats.formatData('editor')
doc = QTextDocument()
font = QFont(data.font)
font.setPointSizeF(font.pointSizeF() * scale)
doc.setDefaultFont(font)
doc.setPlainText(c2.selection().toPlainText())
if metainfo.info(cursor.document()).highlighting:
highlighter.highlight(doc, state=tokeniter.state(block))
size = doc.size().toSize() + QSize(8, -4)
pix = QPixmap(size)
pix.fill(data.baseColors['background'])
doc.drawContents(QPainter(pix))
return pix
def test(self):
from highlighter import highlight_doc as highlight
doc = """A great example of Dlish's genius is their coconut macaroon cupcake: Coconut cake with light almond frosting and toasted coconut drizzled with bittersweet chocolate. Start with the first part - Coconut cake!!! Actual coconut cake, with coconut milk in the batter! The vast majority of cupcakeries in Toronto are making vanilla cupcakes and just adding a different frosting to make their different flavours. Not Dlish - their cupcakes are actually different cakes, not just different frostings.
"""
query = 'delicious dlish cupcakes!'
result = highlight(doc, query)
result = result.lower()
assert 'dlish' in result
assert '[[highlight]]' in result
assert '[[endhighlight]]' in result
doc = """Each cuppie cake is $3 a pop but the vanilla on chocolate is simply put it -dlish! The store is small and classy looking and they bake in store. The icing unlike others is smooth, rich and creamy. And the cake is moist and fresh (almost too beautiful to eat). They have a wide range of flavours which they change up but my favourite is the vanilla icing on chocolate cake. :)
"""
result = highlight(doc, 'gross')
result = result.lower()
assert '[[highlight]]' not in result
assert '[[endhighlight]]' not in result
doc = """LAWD. Best pizza ever. Oh lawd this is my happy place. Anytime I'm anywhere near San Francisco I demand a stop to this place. I was up here the week of my birthday and spent $80 on pizza. We ate two and got one to go, for the trip home. I love you, pizzeria delfina.
Bonus points: Sitting at my table minding my bidness when a handsome dude was strolling down the street, looking very annoying, when I noticed he was carting around a full skeleton in a tote bag, on his shoulder. WTF??!?!
"""
result = highlight(doc, 'pizza birthday')
result = result.lower()
assert 'pizza' in result
assert 'birthday' in result
assert '[[highlight]]' in result
assert '[[endhighlight]]' in result
def show(cursor, pos=None, num_lines=6):
"""Displays a tooltip showing part of the cursor's Document.
If the cursor has a selection, those blocks are displayed.
Otherwise, num_lines lines are displayed.
If pos is not given, the global mouse position is used.
"""
block = cursor.document().findBlock(cursor.selectionStart())
c2 = QTextCursor(block)
if cursor.hasSelection():
c2.setPosition(cursor.selectionEnd(), QTextCursor.KeepAnchor)
c2.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
else:
c2.movePosition(QTextCursor.NextBlock, QTextCursor.KeepAnchor, num_lines)
data = textformats.formatData('editor')
doc = QTextDocument()
font = QFont(data.font)
font.setPointSizeF(font.pointSizeF() * .8)
doc.setDefaultFont(font)
doc.setPlainText(c2.selection().toPlainText())
if metainfo.info(cursor.document()).highlighting:
highlighter.highlight(doc, state=tokeniter.state(block))
size = doc.size().toSize() + QSize(8, -4)
pix = QPixmap(size)
pix.fill(data.baseColors['background'])
doc.drawContents(QPainter(pix))
label = QLabel()
label.setPixmap(pix)
label.setStyleSheet("QLabel { border: 1px solid #777; }")
label.resize(size)
widgets.customtooltip.show(label, pos)
def pixmap(cursor, num_lines=6, scale=0.8):
"""Return a QPixmap displaying the selected lines of the document.
If the cursor has no selection, num_lines are drawn.
By default the text is drawn 0.8 * the normal font size. You can change
that by supplying the scale parameter.
"""
block = cursor.document().findBlock(cursor.selectionStart())
c2 = QTextCursor(block)
if cursor.hasSelection():
c2.setPosition(cursor.selectionEnd(), QTextCursor.KeepAnchor)
c2.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
else:
c2.movePosition(QTextCursor.NextBlock, QTextCursor.KeepAnchor,
num_lines)
data = textformats.formatData('editor')
doc = QTextDocument()
font = QFont(data.font)
font.setPointSizeF(font.pointSizeF() * scale)
doc.setDefaultFont(font)
doc.setPlainText(c2.selection().toPlainText())
if metainfo.info(cursor.document()).highlighting:
highlighter.highlight(doc, state=tokeniter.state(block))
size = doc.size().toSize() + QSize(8, -4)
pix = QPixmap(size)
pix.fill(data.baseColors['background'])
doc.drawContents(QPainter(pix))
return pix
def loadingFinished(self):
data = self._reply.readAll()
self._reply.close()
self._reply.deleteLater()
del self._reply
self.textbrowser.clear()
self.textbrowser.setText(str(data, 'utf-8', 'replace'))
highlighter.highlight(self.textbrowser.document())
def loadingFinished(self):
data = self._reply.readAll()
self._reply.close()
self._reply.deleteLater()
del self._reply
self.textbrowser.clear()
self.textbrowser.setText(str(data, 'utf-8', 'replace'))
highlighter.highlight(self.textbrowser.document())
def varinfo(var):
"""Pretty print information about a variable."""
import numpy
from IPython import get_ipython
from IPython.lib.pretty import pretty
from highlighter import highlight
ip = get_ipython()
if ip:
print(ip.inspector._get_info(var)['text/plain'])
try:
s = pretty(var, max_seq_length=20)
except TypeError:
s = pretty(var)
lines = s.splitlines()
if len(lines) > 20:
s = '\n'.join(lines[:10] + ['...'] + lines[-10:])
print(highlight(s).strip())
print(type(var))
if isinstance(var, numpy.ndarray):
print(var.shape, var.dtype)
print('min = {} max = {}'.format(
numpy.nanmin(var), numpy.nanmax(var)))
print('mean = {} std = {}'.format(
numpy.nanmean(var), numpy.nanstd(var)))
elif isinstance(var, (dict, list, tuple, set)):
print('n = %d' % len(var))
def _nice_tweet(tweet, q_toks, topic_ngrams):
s = ""
s += '<span class="text">'
hl_spec = dict((ng, ('<span class="topic_hl">','</span>')) for ng in topic_ngrams)
for qg in list(set(bigrams.bigrams(q_toks))) + list(set(bigrams.unigrams(q_toks))):
if len(qg)==1 and qg[0] in bigrams.super_stopwords: continue
if len(qg)==1 and any(qg[0] in ng for ng in topic_ngrams): continue
if len(qg)>=2 and any(kmp.isSubseq(qg, ng) for ng in topic_ngrams): continue
hl_spec[qg] = ('<span class="q_hl">','</span>')
text = highlighter.highlight(tweet['toks'], hl_spec)
text = linkify(text, klass='t')
#text = twokenize.Url_RE.subn(r'<a class=t target=_blank href="\1">\1</a>', text)[0]
#text = twokenize.AT_RE.subn(r'<a class=at target=_blank href="\1">\1</a>
text = At.gsub(text, r'<a class="at" target="_blank" href="http://twitter.com/\2">@\2</a>')
s += text
s += "</span>"
s += " "
s += '<span class="authors">'
if 'orig_tweets' in tweet:
s += "%d authors:" % len(tweet['orig_tweets'])
subtweets = tweet['orig_tweets']
else:
subtweets = (tweet,)
for subtweet in subtweets:
user = subtweet['from_user']
link = "http://twitter.com/%s/status/%s" % (user, subtweet['id'])
s += " "
# calling encode() here makes NO SENSE AT ALL why do we need it?
s += '<a class="m" target="_blank" href="%s">%s</a>' % (util.stringify(link), util.stringify(user))
s += '</span>'
return s
def pyin(self, msg):
self.last_execution_count = msg['content']['execution_count']
write('\r')
dots = ' ' * (len(self.print_prompt().rstrip()) - 1) + ': '
code = highlight(msg['content']['code'])
output = code.rstrip().replace('\n', '\n' + colorize(dots, 28))
write(self.wrap(output, prompt=True))
self.execution_count_id = msg['parent_header']['msg_id']
self.last_msg_type = msg['msg_type']
def invalidate_command(self, _=None):
"""
Regenerate both 'lily' and 'oll' versions of the regular and full
font commands. Display the regular command in the textedit and
trigger showing the sample in the music font tab.
"""
self._cmd['lily'], self._full_cmd['lily'] = self.generate_lily_command(
)
self._cmd['oll'], self._full_cmd['oll'] = (self.generate_oll_command())
self.approach = ('lily'
if self.approach_tab.currentIndex() == 0 else 'oll')
display_cmd = self._cmd[self.approach]
self.command_edit.setPlainText(display_cmd)
highlighter.highlight(self.command_edit.document())
for k in self.font_labels:
font_key = 'music' if k == 'oll_music' else k
self.font_labels[k].setText(self.window().selected_font(font_key))
self.window().show_sample()
def pyin(self, msg):
self.last_execution_count = msg['content']['execution_count']
write('\r')
dots = ' ' * (len(self.print_prompt().rstrip()) - 1) + ': '
code = highlight(msg['content']['code'])
output = code.rstrip().replace('\n', '\n' + colorize(dots, 28))
write(self.wrap(output, prompt=True))
self.execution_count_id = msg['parent_header']['msg_id']
self.last_msg_type = msg['msg_type']
def pyin(self, msg):
self.last_execution_count = msg['content']['execution_count']
sys.stdout.write('\r')
dots = '.' * len(self.print_prompt().rstrip()) + ' '
code = highlight(msg['content']['code'])
output = code.rstrip().replace('\n', '\n' + dots)
sys.stdout.write(output)
self.execution_count_id = msg['parent_header']['msg_id']
self.last_msg_type = msg['msg_type']
def pyin(self, msg):
self.last_execution_count = msg['content']['execution_count']
sys.stdout.write('\r')
dots = '.' * len(self.print_prompt().rstrip()) + ' '
code = highlight(msg['content']['code'])
output = code.rstrip().replace('\n', '\n' + dots)
sys.stdout.write(output)
self.execution_count_id = msg['parent_header']['msg_id']
self.last_msg_type = msg['msg_type']
def do_GET(self):
input = urllib2.unquote(self.path[1:])
lang, _, data = input.partition("?")
if not data.startswith("["):
do_404(self)
return
data = json.loads(data)
html, css = highlight(data, lang=lang, output="html", unescape=True)
self.send_response(200)
self.send_header('Content-type', "text/plain")
self.end_headers()
self.wfile.write(html.encode("utf-8"))
def show(cursor, pos=None, num_lines=6):
"""Displays a tooltip showing part of the cursor's Document.
If the cursor has a selection, those blocks are displayed.
Otherwise, num_lines lines are displayed.
If pos is not given, the global mouse position is used.
"""
block = cursor.document().findBlock(cursor.selectionStart())
c2 = QTextCursor(block)
if cursor.hasSelection():
c2.setPosition(cursor.selectionEnd(), QTextCursor.KeepAnchor)
c2.movePosition(QTextCursor.EndOfBlock, QTextCursor.KeepAnchor)
else:
c2.movePosition(QTextCursor.NextBlock, QTextCursor.KeepAnchor,
num_lines)
data = textformats.formatData('editor')
doc = QTextDocument()
font = QFont(data.font)
font.setPointSizeF(font.pointSizeF() * .8)
doc.setDefaultFont(font)
doc.setPlainText(c2.selection().toPlainText())
if metainfo.info(cursor.document()).highlighting:
highlighter.highlight(doc, state=tokeniter.state(block))
size = doc.size().toSize() + QSize(8, -4)
pix = QPixmap(size)
pix.fill(data.baseColors['background'])
doc.drawContents(QPainter(pix))
label = QLabel()
label.setPixmap(pix)
label.setStyleSheet("QLabel { border: 1px solid #777; }")
label.resize(size)
widgets.customtooltip.show(label, pos)
def varinfo(var):
"""Pretty print information about a variable."""
import numpy
from IPython.lib.pretty import pretty
from highlighter import highlight
try:
s = pretty(var, max_seq_length=20)
except TypeError:
s = pretty(var)
lines = s.splitlines()
if len(lines) > 20:
s = '\n'.join(lines[:10] + ['...'] + lines[-10:])
print(highlight(s).strip())
print(type(var))
if isinstance(var, numpy.ndarray):
print(var.shape, var.dtype)
elif isinstance(var, (dict, list, tuple, set)):
print('n = %d' % len(var))
def grep(file, regex, highlight_text):
"""
Takes a file and finds all matches based on what the user chooses to match
Args:
file (Str): Text from the file from which to find matches to
regex (List): Chosen words or "regexes" to use to find matches in the text
highlight_text (Boolean): Used as flag if you chooses to highlight matching text
"""
color_list = ['38;5;13', '38;5;44', '38;5;229']
regex_list = regex
sb = ""
regex_color_dict = {}
index = 0
for r in regex_list:
# To cycle through colors
if index == len(color_list):
index = 0
# Setting regex with color
regex_color_dict[r] = color_list[index]
index += 1
new_regex = "^.*" + r + ".*$"
matches = re.finditer(new_regex, file, re.M)
for match in matches:
line = match.group() + "\n"
# Will ignore duplicates
if sb.find(line) == -1:
sb += line
# If flag for highlight is true, use highlight-method from previous task
if highlight_text:
result = highlight(regex_color_dict, sb)
print(result)
else:
print(sb)
def varinfo(var):
"""Pretty print information about a variable."""
import numpy
from IPython.lib.pretty import pretty
from highlighter import highlight
try:
s = pretty(var, max_seq_length=20)
except TypeError:
s = pretty(var)
lines = s.splitlines()
if len(lines) > 20:
s = '\n'.join(lines[:10] + ['...'] + lines[-10:])
print(highlight(s).strip())
print(type(var))
if isinstance(var, numpy.ndarray):
print(var.shape, var.dtype)
print('min = {} max = {}'.format(
numpy.nanmin(var), numpy.nanmax(var)))
print('mean = {} std = {}'.format(
numpy.nanmean(var), numpy.nanstd(var)))
elif isinstance(var, (dict, list, tuple, set)):
print('n = %d' % len(var))
def _nice_tweet(tweet, q_toks, topic_ngrams):
s = ""
s += '<span class="text">'
hl_spec = dict(
(ng, ('<span class="topic_hl">', '</span>')) for ng in topic_ngrams)
for qg in list(set(bigrams.bigrams(q_toks))) + list(
set(bigrams.unigrams(q_toks))):
if len(qg) == 1 and qg[0] in bigrams.super_stopwords: continue
if len(qg) == 1 and any(qg[0] in ng for ng in topic_ngrams): continue
if len(qg) >= 2 and any(kmp.isSubseq(qg, ng) for ng in topic_ngrams):
continue
hl_spec[qg] = ('<span class="q_hl">', '</span>')
text = highlighter.highlight(tweet['toks'], hl_spec)
text = linkify(text, klass='t')
#text = twokenize.Url_RE.subn(r'<a class=t target=_blank href="\1">\1</a>', text)[0]
#text = twokenize.AT_RE.subn(r'<a class=at target=_blank href="\1">\1</a>
text = At.gsub(
text,
r'<a class="at" target="_blank" href="http://twitter.com/\2">@\2</a>')
s += text
s += "</span>"
s += " "
s += '<span class="authors">'
if 'orig_tweets' in tweet:
s += "%d authors:" % len(tweet['orig_tweets'])
subtweets = tweet['orig_tweets']
else:
subtweets = (tweet, )
for subtweet in subtweets:
user = subtweet['from_user']
link = "http://twitter.com/%s/status/%s" % (user, subtweet['id'])
s += " "
# calling encode() here makes NO SENSE AT ALL why do we need it?
s += '<a class="m" target="_blank" href="%s">%s</a>' % (
util.stringify(link), util.stringify(user))
s += '</span>'
return s
def readSettings(self):
data = textformats.formatData('editor')
self.textbrowser.setPalette(data.palette())
self.textbrowser.setFont(data.font)
highlighter.highlight(self.textbrowser.document())
def diff(og, mod, color):
if color:
f = open("demo.colordiff_espenlon.txt", "w+")
mod_lines = []
for line in mod:
line = line[:-1]
mod_lines.append(line)
og_lines = []
for line in og:
line = line[:-1]
og_lines.append(line)
mod_index = 0
og_index = 0
while og_index < len(og_lines) or mod_index < len(mod_lines):
mod_inc = 1
og_inc = 1
if og_index >= len(og_lines):
if color:
f.write("+" + mod_lines[mod_index] + "\n")
else:
print("+" + mod_lines[mod_index])
elif mod_index >= len(mod_lines):
if color:
f.write("-" + og_lines[og_index] + "\n")
else:
print("-" + og_lines[og_index])
else:
if og_lines[og_index] == mod_lines[mod_index]:
if color:
f.write("0" + og_lines[og_index] + "\n")
else:
print("0" + og_lines[og_index])
elif og_index + 1 < len(og_lines) and og_lines[
og_index + 1] == mod_lines[mod_index]:
if color:
f.write("-" + og_lines[og_index] + "\n")
f.write("0" + mod_lines[mod_index] + "\n")
og_inc = 2
else:
print("-" + og_lines[og_index])
print("0" + mod_lines[mod_index])
og_inc = 2
elif mod_index + 1 < len(
mod_lines) and og_lines[og_index] == mod_lines[mod_index +
1]:
if color:
f.write("+" + mod_lines[mod_index] + "\n")
f.write("0" + og_lines[og_index] + "\n")
mod_inc = 2
else:
print("+" + mod_lines[mod_index])
print("0" + og_lines[og_index])
mod_inc = 2
else:
if color:
f.write("-" + og_lines[og_index] + "\n")
f.write("+" + mod_lines[mod_index] + "\n")
else:
print("-" + og_lines[og_index])
print("+" + mod_lines[mod_index])
og_index += og_inc
mod_index += mod_inc
if color:
f.close()
syntax_file = open("diff.syntax", "r")
theme_file = open("diff.theme", "r")
program_file = open("demo.colordiff_espenlon.txt", "r")
syntax = read_syntax_file(syntax_file)
theme = read_theme_file(theme_file)
highlighted_file = highlight(program_file, syntax, theme)
print(highlighted_file)
def highlight(file):
for line in highlighter.highlight(file, patterns, colors):
sys.stdout.write(line)
def readSettings(self):
data = textformats.formatData('editor')
self.textbrowser.setPalette(data.palette())
self.textbrowser.setFont(data.font)
highlighter.highlight(self.textbrowser.document())
def home():
'''display main page (only page, for now, this is a single-page application)'''
kwcolors,keywords,filename = dict(),dict(),str()
if not 'uimode' in session: session['uimode'] = 'simple'
admin= False
scrollpos='0'
if 'username' in session:
if session['username']=='admin':admin=True
db = get_db()
res=db.cursor().execute('select * from keywords where owner = ?',(session['username'],)).fetchall()
kwcolors = {r[1]:r[2] for r in res}
#kwcolors[7] = '00dd00'
print('KWCOLORS:',kwcolors)
keywords = {r[1]:r[3].lower() for r in res}
filename = session['filename']
scrollpos = session['scrollpos']
if 'files' in session and 'username' in session:
t1=time.time()
cboxes = res=db.cursor().execute('select cboxes from users where username = ?',(session['username'],)).fetchone()[0]
#print('CBOXES:',cboxes)
corpus = highlighter.highlight(session['username'],keywords=keywords,uimode=session['uimode'], cboxes=cboxes)
if not corpus:
corpus = 'No document added yet. Please add one.<br>(you can use the \'fetch document\' button above)'
#print('highlight time:',time.time()-t1)
corpus = corpus.replace('\n','<br>')
#print('corpus::: ', corpus)
elif 'username' in session:
corpus = 'No document added yet. Please add one.<br>(you can use the \'fetch document\' button above)'
else:
corpus = '''please login or create an account<br><br>
<h3>advisory:</h3>
when creating an account, please choose a non-personally-identifying username
<br><br>e.g. not your real name or email address
<br><br><a href="/help">site help</a>'''
mode = session['mode'] if 'mode' in session else ''
if session['uimode'] == 'simple':
template = 'simplehome.html'
else:
template = 'home.html'
EU = nsettings.EU
icons = {'pastbreach':0, 'international':0, 'gdpr':0, 'arbitration':0, 'other':0}
try:
if 'files' in session and 'username' in session:
with open('uploads/%s.corpusfile.txt' %session['username'],'r') as f:
text=f.read().lower()
if 'anonym' in text: icons['other']=1
if 'gdpr' in text or 'general data protection' in text: icons['gdpr']=1
if 'arbitration' in text: icons['arbitration']=1
domain = ''
if session['filename'][:10] == 'text from ':
domain = session['filename'][11:]
if 'http' in session['filename']:
tld = tldextract.extract(session['filename'])
domain = tld.domain+'.'+tld.suffix
if domain:
hostip= socket.gethostbyname(domain)
z=find_country(hostip)
if not z.lower() in EU: icons['international']=1
if domain in nsettings.breachdomains: icons['pastbreach']=1
except Exception as e:
print('exception: ',e)
pass
return render_template(template,admin=admin,corpus=corpus, scrollpos=scrollpos,
loggedin=loggedin(), kwcolors=kwcolors, mode=mode, keywords=keywords, filename=filename,
icons=icons,
pro_upgrade = session['pro_upgrade'] if 'pro_upgrade' in session else 0)
