class SummaryPage(parentpage):
def __init__(self, parent):
super(SummaryPage, self).__init__(parent, 'Selection Summary')
def GetMyControls(self):
vbs = wx.StaticBoxSizer(wx.StaticBox(self, -1, 'Selection Summary:'),
wx.VERTICAL)
self.summary = ExpandoTextCtrl(self, -1, '')
self.summary.SetBackgroundColour(self.GetBackgroundColour())
vbs.Add(self.summary, 0, wx.EXPAND | wx.ALL, 5)
self._box = vbs
return vbs
def set(self, data):
text = ['Name:\t%s' % data.get('name', '')]
text.append('Dir:\t%s' % data.get('path', ''))
if data.get('currentsettings', False):
text.append('Use current BitPim settings.')
else:
text.append('Use default BitPim settings.')
if guihelper.IsMSWindows():
if data.get('desktop', False):
text.append('Create a shortcut on your Desktop.')
if data.get('startmenu', False):
text.append('Create a shortcut in your Start Menu.')
self.summary.SetValue('\n\n'.join(text))
class PathPage(parentpage):
def __init__(self, parent):
super(PathPage, self).__init__(parent, 'Select New Storage Dir')
if guihelper.IsMSWindows():
shell = client.Dispatch("WScript.Shell")
self.defaultdir = os.path.join(shell.SpecialFolders("MyDocuments"),
'Phones')
else:
self.defaultdir = os.path.expanduser('~/Phones')
def GetMyControls(self):
vbs = wx.BoxSizer(wx.VERTICAL)
vbs.Add(wx.StaticText(self, -1, 'Storage Dir:'), 0, wx.EXPAND | wx.ALL,
5)
self.path = ExpandoTextCtrl(self, -1, '', style=wx.TE_READONLY)
self.path.SetBackgroundColour(self.GetBackgroundColour())
vbs.Add(self.path, 0, wx.EXPAND | wx.ALL, 5)
btn = wx.Button(self, -1, 'Browse')
wx.EVT_BUTTON(self, btn.GetId(), self.OnBrowse)
vbs.Add(btn, 0, wx.ALL, 5)
return vbs
def ok(self):
return bool(self.path.GetValue())
def get(self, data):
data['path'] = self.path.GetValue()
def set(self, data):
path = data.get('path', '')
if not path:
path = os.path.join(self.defaultdir, data.get('name', ''))
self.path.SetValue(path)
def OnBrowse(self, _):
with guihelper.WXDialogWrapper(
wx.DirDialog(self,
defaultPath=self.path.GetLabel(),
style=wx.DD_NEW_DIR_BUTTON),
True) as (dlg, retcode):
if retcode == wx.ID_OK:
self.path.SetValue(dlg.GetPath())
class ImportSourcePage(setphone_wizard.MyPage):
def __init__(self, parent):
self._source = None
super(ImportSourcePage, self).__init__(parent, 'Select Import Source')
def GetMyControls(self):
vbs = wx.BoxSizer(wx.VERTICAL)
vbs.Add(wx.StaticText(self, -1, 'Source of data:'), 0,
wx.ALL | wx.EXPAND, 5)
self._source_lbl = ExpandoTextCtrl(self, -1, '', style=wx.TE_READONLY)
self._source_lbl.SetBackgroundColour(self.GetBackgroundColour())
vbs.Add(self._source_lbl, 0, wx.ALL | wx.EXPAND, 5)
_btn = wx.Button(self, -1, 'Browse')
wx.EVT_BUTTON(self, _btn.GetId(), self._OnBrowse)
vbs.Add(_btn, 0, wx.ALL, 5)
return vbs
def setlabel(self):
self._source_lbl.SetValue(self._source.name())
@guihelper.BusyWrapper
def _OnBrowse(self, _=None):
if not self._source:
return
self._source.browse(self)
self.setlabel()
def ok(self):
return self._source and self._source.get()
def get(self, data):
data['source_obj'] = self._source
data['source_id'] = self._source.id
data['imported'] = False
def set(self, data):
self._source = data['source_obj']
if self._source:
if data.has_key('source_id'):
self._source.id = data['source_id']
self.setlabel()
def GetActiveDatabase(self):
return self.GetParent().GetActiveDatabase()
def addRow(self, row, rowLabel=None):
"""Add one row of info, either header (col names) or normal data
Adds items sequentially; FlexGridSizer moves to next row automatically
"""
labelBox = wx.BoxSizer(wx.HORIZONTAL)
if not rowLabel:
if sys.platform == 'darwin':
self.SetWindowVariant(variant=wx.WINDOW_VARIANT_SMALL)
label = _translate('cond %s:') % str(row + 1 -
int(self.hasHeader)).zfill(2)
rowLabel = wx.StaticText(self, -1, label=label)
rowLabel.SetForegroundColour(darkgrey)
if sys.platform == 'darwin':
self.SetWindowVariant(variant=wx.WINDOW_VARIANT_NORMAL)
labelBox.Add(rowLabel, 1, flag=wx.ALIGN_RIGHT | wx.ALIGN_BOTTOM)
self.sizer.Add(labelBox, 1, flag=wx.ALIGN_CENTER)
lastRow = []
for col in range(self.cols):
# get the item, as unicode for display purposes:
if len(str(self.grid[row][col])): # want 0, for example
item = str(self.grid[row][col])
else:
item = u''
# make a textbox:
field = ExpandoTextCtrl(self,
-1,
item,
size=(self.colSizes[col], 20))
field.Bind(EVT_ETC_LAYOUT_NEEDED, self.onNeedsResize)
field.SetMaxHeight(100) # ~ 5 lines
if self.hasHeader and row == 0:
# add a default column name (header) if none provided
header = self.grid[0]
if item.strip() == '':
c = col
while self.colName(c) in header:
c += 1
field.SetValue(self.colName(c))
field.SetForegroundColour(darkblue) # dark blue
# or (self.parent and
if not valid_var_re.match(field.GetValue()):
# self.parent.exp.namespace.exists(field.GetValue()) ):
# was always red when preview .xlsx file -- in
# namespace already is fine
if self.fixed:
field.SetForegroundColour("Red")
field.SetToolTip(
wx.ToolTip(
_translate(
'Should be legal as a variable name (alphanumeric)'
)))
field.Bind(wx.EVT_TEXT, self.checkName)
elif self.fixed:
field.SetForegroundColour(darkgrey)
field.SetBackgroundColour(white)
# warn about whitespace unless will be auto-removed. invisible,
# probably spurious:
if (self.fixed or not self.clean) and item != item.strip():
field.SetForegroundColour('Red')
# also used in show():
self.warning = _translate('extra white-space')
field.SetToolTip(wx.ToolTip(self.warning))
if self.fixed:
field.Disable()
lastRow.append(field)
self.sizer.Add(field, 1)
self.inputFields.append(lastRow)
if self.hasHeader and row == 0:
self.header = lastRow
class AbstractEngineSettingsPanel(SettingsPanel):
"""
Settings panel of external services.
ocrHandler must be specified before use.
"""
# Developers: Please also specify a comment for translators
name = _("Engine")
title = _("Engine")
descEngineNameCtrl = None # type: ExpandoTextCtrl
engineSettingPanel = None # type: SpecificEnginePanel
handler = AbstractEngineHandler # type: AbstractEngineHandler
def makeGeneralSettings(self, settingsSizerHelper):
"""
Generate general settings for engine ocrHandler
@param settingsSizerHelper:
@type settingsSizerHelper:
"""
pass
def makeSettings(self, settingsSizer):
settingsSizerHelper = guiHelper.BoxSizerHelper(self,
sizer=settingsSizer)
# Translators: A label for the engines on the engine panel.
engineLabel = self.title + _(" Engines")
engineBox = wx.StaticBox(self, label=engineLabel)
engineGroup = guiHelper.BoxSizerHelper(self,
sizer=wx.StaticBoxSizer(
engineBox, wx.HORIZONTAL))
settingsSizerHelper.addItem(engineGroup)
# Use a ExpandoTextCtrl because even when readonly it accepts focus from keyboard, which
# standard readonly TextCtrl does not. ExpandoTextCtrl is a TE_MULTILINE control, however
# by default it renders as a single line. Standard TextCtrl with TE_MULTILINE has two lines,
# and a vertical scroll bar. This is not necessary for the single line of text we wish to
# display here.
engine = self.handler.getCurrentEngine()
engineDesc = engine.description
msg = "Engine:\n{0}\nDescription:\n{1}\n".format(engine, engineDesc)
log.debug(msg)
self.descEngineNameCtrl = ExpandoTextCtrl(self,
size=(self.scaleSize(250),
-1),
value=engineDesc,
style=wx.TE_READONLY)
self.descEngineNameCtrl.Bind(wx.EVT_CHAR_HOOK,
self._enterTriggersOnChangeEngine)
# Translators: This is the label for the button used to change engines,
# it appears in the context of a engine group on the Online OCR settings panel.
changeEngineBtn = wx.Button(self, label=_("C&hange..."))
engineGroup.addItem(
guiHelper.associateElements(self.descEngineNameCtrl,
changeEngineBtn))
changeEngineBtn.Bind(wx.EVT_BUTTON, self.onChangeEngine)
self.engineSettingPanel = SpecificEnginePanel(self, self.handler)
settingsSizerHelper.addItem(self.engineSettingPanel)
self.makeGeneralSettings(settingsSizerHelper)
def _enterTriggersOnChangeEngine(self, evt):
if evt.KeyCode == wx.WXK_RETURN:
self.onChangeEngine(evt)
else:
evt.Skip()
def onChangeEngine(self, evt):
change_engine = EnginesSelectionDialog(self,
self.handler,
multiInstanceAllowed=True)
ret = change_engine.ShowModal()
if ret == wx.ID_OK:
self.Freeze()
# trigger a refresh of the settings
self.onPanelActivated()
self._sendLayoutUpdatedEvent()
self.Thaw()
def updateCurrentEngine(self):
engine_description = self.handler.getCurrentEngine().description
self.descEngineNameCtrl.SetValue(engine_description)
def onPanelActivated(self):
# call super after all panel updates have been completed, we do not want the panel to show until this is complete.
self.engineSettingPanel.onPanelActivated()
super(AbstractEngineSettingsPanel, self).onPanelActivated()
def onPanelDeactivated(self):
self.engineSettingPanel.onPanelDeactivated()
super(AbstractEngineSettingsPanel, self).onPanelDeactivated()
def onDiscard(self):
self.engineSettingPanel.onDiscard()
def onSave(self):
self.engineSettingPanel.onSave()
class DialogAcknowledge(wx.Dialog):
"""
"Acknowledgement" Dialog Box
Shows the current method for acknowledging SasView in
scholarly publications.
"""
def __init__(self, *args, **kwds):
kwds["style"] = wx.DEFAULT_DIALOG_STYLE
wx.Dialog.__init__(self, *args, **kwds)
self.ack = ExpandoTextCtrl(self, style=wx.TE_LEFT|wx.TE_MULTILINE|wx.TE_BESTWRAP|wx.TE_READONLY|wx.TE_NO_VSCROLL)
self.ack.SetValue(config._acknowledgement_publications)
#self.ack.SetMinSize((-1, 55))
self.citation = ExpandoTextCtrl(self, style=wx.TE_LEFT|wx.TE_MULTILINE|wx.TE_BESTWRAP|wx.TE_READONLY|wx.TE_NO_VSCROLL)
self.citation.SetValue(config._acknowledgement_citation)
self.preamble = wx.StaticText(self, -1, config._acknowledgement_preamble)
items = [config._acknowledgement_preamble_bullet1,
config._acknowledgement_preamble_bullet2,
config._acknowledgement_preamble_bullet3,
config._acknowledgement_preamble_bullet4]
self.list1 = wx.StaticText(self, -1, "(1) " + items[0])
self.list2 = wx.StaticText(self, -1, "(2) " + items[1])
self.list3 = wx.StaticText(self, -1, "(3) " + items[2])
self.list4 = wx.StaticText(self, -1, "(4) " + items[3])
self.static_line = wx.StaticLine(self, 0)
self.__set_properties()
self.__do_layout()
def __set_properties(self):
"""
:TODO - add method documentation
"""
# begin wxGlade: DialogAbout.__set_properties
self.preamble.SetFont(wx.Font(11, wx.DEFAULT, wx.NORMAL, wx.NORMAL, 0, ""))
self.preamble.SetFont(wx.Font(10, wx.DEFAULT, wx.NORMAL, wx.NORMAL, 0, ""))
self.SetTitle("Acknowledging SasView")
#Increased size of box from (525, 225), SMK, 04/10/16
self.SetClientSize((600, 320))
# end wxGlade
def __do_layout(self):
"""
:TODO - add method documentation
"""
# begin wxGlade: DialogAbout.__do_layout
sizer_main = wx.BoxSizer(wx.VERTICAL)
sizer_titles = wx.BoxSizer(wx.VERTICAL)
sizer_titles.Add(self.preamble, 0, wx.ALL|wx.EXPAND, 5)
sizer_titles.Add(self.list1, 0, wx.ALL|wx.EXPAND, 5)
sizer_titles.Add(self.ack, 0, wx.ALL|wx.EXPAND, 5)
sizer_titles.Add(self.list2, 0, wx.ALL|wx.EXPAND, 5)
sizer_titles.Add(self.citation, 0, wx.ALL|wx.EXPAND, 5)
sizer_titles.Add(self.list3, 0, wx.ALL|wx.EXPAND, 5)
#sizer_titles.Add(self.static_line, 0, wx.ALL|wx.EXPAND, 0)
sizer_titles.Add(self.list4, 0, wx.ALL|wx.EXPAND, 5)
sizer_main.Add(sizer_titles, -1, wx.ALL|wx.EXPAND, 5)
self.SetAutoLayout(True)
self.SetSizer(sizer_main)
self.Layout()
self.Centre()
class TestFrame(wx.Frame):
def __init__(self, parent, log):
wx.Frame.__init__(self, parent, title="Test ExpandoTextCtrl")
self.log = log
self.pnl = p = wx.Panel(self)
self.eom = ExpandoTextCtrl(
p, size=(250, -1), value="This control will expand as you type")
self.Bind(EVT_ETC_LAYOUT_NEEDED, self.OnRefit, self.eom)
# create some buttons and sizers to use in testing some
# features and also the layout
vBtnSizer = wx.BoxSizer(wx.VERTICAL)
btn = wx.Button(p, -1, "Set MaxHeight")
self.Bind(wx.EVT_BUTTON, self.OnSetMaxHeight, btn)
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
btn = wx.Button(p, -1, "Set Font")
self.Bind(wx.EVT_BUTTON, self.OnSetFont, btn)
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
btn = wx.Button(p, -1, "Write Text")
self.Bind(wx.EVT_BUTTON, self.OnWriteText, btn)
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
btn = wx.Button(p, -1, "Append Text")
self.Bind(wx.EVT_BUTTON, self.OnAppendText, btn)
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
btn = wx.Button(p, -1, "Set Value")
self.Bind(wx.EVT_BUTTON, self.OnSetValue, btn)
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
btn = wx.Button(p, -1, "Get Value")
self.Bind(wx.EVT_BUTTON, self.OnGetValue, btn)
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
for x in range(3):
btn = wx.Button(p, -1, " ")
vBtnSizer.Add(btn, 0, wx.ALL | wx.EXPAND, 5)
self.Bind(wx.EVT_BUTTON, self.OnOtherBtn, btn)
hBtnSizer = wx.BoxSizer(wx.HORIZONTAL)
for x in range(3):
btn = wx.Button(p, -1, " ")
hBtnSizer.Add(btn, 0, wx.ALL, 5)
self.Bind(wx.EVT_BUTTON, self.OnOtherBtn, btn)
sizer = wx.BoxSizer(wx.HORIZONTAL)
col1 = wx.BoxSizer(wx.VERTICAL)
col1.Add(self.eom, 0, wx.ALL, 10)
col1.Add(hBtnSizer)
sizer.Add(col1)
sizer.Add(vBtnSizer)
p.SetSizer(sizer)
# Put the panel in a sizer for the frame so we can use self.Fit()
frameSizer = wx.BoxSizer()
frameSizer.Add(p, 1, wx.EXPAND)
self.SetSizer(frameSizer)
self.Fit()
def OnRefit(self, evt):
# The Expando control will redo the layout of the
# sizer it belongs to, but sometimes this may not be
# enough, so it will send us this event so we can do any
# other layout adjustments needed. In this case we'll
# just resize the frame to fit the new needs of the sizer.
self.Fit()
def OnSetMaxHeight(self, evt):
mh = self.eom.GetMaxHeight()
dlg = wx.NumberEntryDialog(self, "", "Enter new max height:",
"MaxHeight", mh, -1, 1000)
if dlg.ShowModal() == wx.ID_OK:
self.eom.SetMaxHeight(dlg.GetValue())
dlg.Destroy()
def OnSetFont(self, evt):
dlg = wx.FontDialog(self, wx.FontData())
dlg.GetFontData().SetInitialFont(self.eom.GetFont())
if dlg.ShowModal() == wx.ID_OK:
self.eom.SetFont(dlg.GetFontData().GetChosenFont())
dlg.Destroy()
def OnWriteText(self, evt):
self.eom.WriteText("\nThis is a test... Only a test. If this had "
"been a real emergency you would have seen the "
"quick brown fox jump over the lazy dog.\n")
def OnAppendText(self, evt):
self.eom.AppendText("\nAppended text.")
def OnSetValue(self, evt):
self.eom.SetValue("A new value.")
def OnGetValue(self, evt):
self.log.write("-----------------\n" + self.eom.GetValue())
def OnOtherBtn(self, evt):
# just for testing...
#print(self.eom.numLines)
self.eom._adjustCtrl()
class JSCleaner(wx.Frame):
def __init__(self, parent, id, title):
wx.Frame.__init__(self,parent,id,title,size=(800,800), style=wx.DEFAULT_FRAME_STYLE ^ wx.RESIZE_BORDER)
self.number_of_buttons = 2
self.vbox = wx.BoxSizer(wx.VERTICAL)
self.display = wx.TextCtrl(self, style=wx.TE_LEFT)
# self.display.SetValue("http://yasirzaki.net")
self.display.SetValue("http://www.irs.gov")
self.vbox.Add(self.display, flag=wx.EXPAND|wx.TOP|wx.BOTTOM, border=4)
my_btn = wx.Button(self, label='Analyze page')
my_btn.Bind(wx.EVT_BUTTON, self.on_press)
self.vbox.Add(my_btn, 0, wx.ALL | wx.CENTER, 5)
self.textBox = ExpandoTextCtrl (self)
self.vbox.Add(self.textBox, 0, wx.EXPAND, 5)
self.Bind(EVT_ETC_LAYOUT_NEEDED, self.OnRefit, self.textBox)
self.number_of_buttons += 1
self.SetSizer(self.vbox)
self.url = ""
def on_script_press(self, event):
try:
name = event.GetEventObject().myname
toggle = event.GetEventObject().GetValue()
except:
name = "script0"
toggle = True
if toggle:
self.textBox.SetValue(self.JavaScripts[name][0])
print("\n")
print("-"*20)
os.system("clear")
print (jsbeautifier.beautify(self.JavaScripts[name][1]))
# print (self.JavaScripts[name][2])
self.html = self.html.replace("<!--"+name+"-->",self.JavaScripts[name][2])
self.encode_save_index (self.html, "irs.gov", os.getcwd() + "/../proxy/data/")
driver.get(self.url + "/js.html")
else:
self.selectAll.SetValue(False)
self.textBox.SetValue("")
os.system("clear")
self.html = self.html.replace(self.JavaScripts[name][2], "<!--"+name+"-->")
self.encode_save_index (self.html, "irs.gov", os.getcwd() + "/../proxy/data/")
driver.get(self.url + "/js.html")
def on_all_press(self, event):
try:
toggle = event.GetEventObject().GetValue()
except:
toggle = True
if toggle:
# Insert all scripts
for name in self.JavaScripts:
if "<!--"+name+"-->" in self.html:
self.html = self.html.replace("<!--"+name+"-->", self.JavaScripts[name][2])
self.encode_save_index (self.html, "irs.gov", os.getcwd() + "/../proxy/data/")
driver.get(self.url + "/js.html")
# Toggle all script buttons
for btn in self.scriptButtons:
btn.SetValue(True)
else:
# Remove all scripts
for name in self.JavaScripts:
if self.JavaScripts[name][2] in self.html:
self.html = self.html.replace(self.JavaScripts[name][2], "<!--"+name+"-->")
self.encode_save_index (self.html, "irs.gov", os.getcwd() + "/../proxy/data/")
driver.get(self.url + "/js.html")
# Untoggle all script buttons
for btn in self.scriptButtons:
btn.SetValue(False)
def on_press(self, event):
self.url = self.display.GetValue()
if not self.url:
return
self.JavaScripts = {}
self.scriptButtons = []
driver.get(self.url)
html_source = driver.page_source
self.html = str(BeautifulSoup(html_source, 'html.parser'))
#Here is the part which extracts Scripts
scripts = driver.find_elements_by_tag_name("script")
scriptsCount = self.html.count("<script")
self.selectAll = wx.ToggleButton(self, label='Select All')
self.selectAll.Bind(wx.EVT_TOGGLEBUTTON, self.on_all_press)
self.vbox.Add(self.selectAll, 0, wx.ALIGN_LEFT | wx.ALL, 5)
self.number_of_buttons += 1
self.panel = wx.lib.scrolledpanel.ScrolledPanel(self,-1, size=(600,700), style=wx.SIMPLE_BORDER) #pos=(20,100)
self.panel.SetupScrolling()
self.panel.SetBackgroundColour('#FFFFFF')
self.vbox.Add(self.panel, 0, wx.EXPAND, 5)
self.SetSizer(self.vbox)
self.gs = wx.GridSizer(scriptsCount,4,5,5)
cnt = 0
firstButton = False
while "<script" in self.html:
sIndex = self.html.find("<script")
eIndex = self.html.find("</script>")
text = self.html[sIndex:eIndex+9]
if ' src="' in text:
src = text.split(' src=')[1].split('"')[1].replace("http://","").replace("https://","")
src = src.split("?")[0]
contentText = ""
# Connect to the database.
conn = pymysql.connect(db=db_name,user=db_user,passwd=db_password,host='localhost',autocommit=True)
d = conn.cursor()
sql = "SELECT filename FROM caching WHERE url LIKE '%{0}%'".format(src)
d.execute(sql)
if d.rowcount > 0:
filename = d.fetchone()[0]
contentText = getScriptText(filename)
else:
src = src.strip("/").split("/")
src[0] = src[0]+":443"
src = "/".join(src)
sql = "SELECT filename FROM caching WHERE url LIKE '%{0}%'".format(src)
d.execute(sql)
if d.rowcount > 0:
filename = d.fetchone()[0]
contentText = getScriptText(filename)
else:
print (d.rowcount, src)
print (text)
print (contentText[:200])
print ("---"*20)
d.close()
conn.close()
else:
contentText = text
self.html = self.html.replace(text,"\n<!--script"+str(cnt)+"-->\n")
self.scriptButtons.append(wx.ToggleButton(self.panel, label="script"+str(cnt), size=(100,50)))
self.scriptButtons[cnt].Bind(wx.EVT_TOGGLEBUTTON, self.on_script_press)
self.scriptButtons[cnt].myname = "script"+str(cnt)
self.gs.Add(self.scriptButtons[cnt], 0, wx.ALL, 0)
if firstButton == False:
firstButton = self.scriptButtons[cnt]
labels = ["critical","non-critical","translatable"]
colors = [wx.Colour(255, 0, 0),wx.Colour(0, 255, 0),wx.Colour(0, 0, 255)]
for i in range(3):
textBox = wx.ToggleButton(self.panel, label=labels[i], size=(100,25))
textBox.SetBackgroundColour(colors[i])
textBox.SetForegroundColour(colors[i])
self.gs.Add(textBox, 0, wx.ALL,0)
tmp = {}
for feature in features:
if feature in contentText:
tmp[feature] = contentText.count(feature)
tmp_sorted = OrderedDict(sorted(tmp.items(), key=lambda x: x[1], reverse=True))
tmp = ""
for k, v in tmp_sorted.items():
tmp += "{0}: {1}\n".format(k,v)
self.JavaScripts["script"+str(cnt)] = [tmp, contentText, text]
self.number_of_buttons += 1
cnt += 1
self.panel.SetSizer(self.gs)
self.textBox.SetValue("Feature display will be here\n\n\n\n\n")
self.encode_save_index (self.html, "irs.gov", os.getcwd() + "/../proxy/data/")
driver.get(self.url + "/js.html")
def encode_save_index(self, content, name, path):
with gzip.open(path + name + ".c", "wb") as f:
f.write(content.encode())
f.close
print ("HTML is encoded and saved!")
content_size = os.path.getsize(path + name + ".c")
with open(path + name + ".h") as f:
new_text = ""
existing_size = ""
for line in f:
if "Content-Length:" in line:
existing_size = line.split(' ',1)[1]
if(existing_size != ""):
with open(path + name + ".h") as f:
atext = f.read().replace(existing_size, str(content_size)+ "\n")
with open(path + name + ".h", "w") as f:
f.write(atext)
def OnRefit(self, evt):
self.panel.SetSizer(self.gs)
class MyPanel(wx.Panel):
def __init__(self, parent):
wx.Panel.__init__(self, parent)
self.number_of_buttons = 0
self.colors = {
"": wx.Colour(255, 255, 255, 100),
"critical": wx.Colour(255, 0, 0, 100),
"non-critical": wx.Colour(0, 255, 0, 100),
"translatable": wx.Colour(0, 0, 255, 100)
}
self.frame = parent
self.fileName = ""
self.mainSizer = wx.BoxSizer(wx.VERTICAL)
self.url_input = wx.TextCtrl(self, style=wx.TE_LEFT)
self.url_input.SetValue("http://www.yasirzaki.net")
self.url_input.Bind(wx.EVT_KEY_DOWN, self.on_key_press)
self.mainSizer.Add(self.url_input,
flag=wx.EXPAND | wx.TOP | wx.LEFT | wx.RIGHT,
border=25)
# StaticText field for error messages
self.err_msg = wx.StaticText(self, label="")
self.err_msg.SetForegroundColour((255, 0, 0)) # make text red
self.mainSizer.Add(self.err_msg, 0, flag=wx.LEFT, border=25)
analyze_btn = wx.Button(self, label='Analyze page')
analyze_btn.Bind(wx.EVT_BUTTON, self.on_analyze_press)
self.mainSizer.Add(analyze_btn, 0, wx.ALL | wx.CENTER, 25)
self.scripts_panel = wx.lib.scrolledpanel.ScrolledPanel(self,
-1,
size=(750,
400))
self.scripts_panel.SetupScrolling()
# self.scripts_panel.Hide()
# self.scripts_panel.SetBackgroundColour('#FFFFFF')
self.mainSizer.Add(self.scripts_panel, 0, wx.CENTER | wx.BOTTOM, 25)
self.select_all_btn = wx.ToggleButton(self, label='Select All')
self.select_all_btn.Bind(wx.EVT_TOGGLEBUTTON, self.on_all_press)
self.select_all_btn.Hide()
self.mainSizer.Add(self.select_all_btn, 0, wx.BOTTOM | wx.CENTER, 25)
self.diff_btn = wx.Button(self, label='Get diff')
self.diff_btn.Bind(wx.EVT_BUTTON, self.on_diff_press)
self.diff_btn.Hide()
self.mainSizer.Add(self.diff_btn, 0, wx.BOTTOM | wx.CENTER, 25)
vbox = wx.BoxSizer(wx.HORIZONTAL)
self.features_panel = wx.lib.scrolledpanel.ScrolledPanel(
self, -1, size=(375, 300), style=wx.SIMPLE_BORDER)
self.features_panel.SetupScrolling()
self.features_panel.SetBackgroundColour('#FFFFFF')
vbox.Add(self.features_panel, 0, wx.CENTER, 5)
self.content_panel = wx.lib.scrolledpanel.ScrolledPanel(
self, -1, size=(375, 300), style=wx.SIMPLE_BORDER)
self.content_panel.SetupScrolling()
self.content_panel.SetBackgroundColour('#FFFFFF')
vbox.Add(self.content_panel, 0, wx.CENTER, 5)
self.mainSizer.Add(vbox, 0, wx.CENTER, 5)
self.SetSizer(self.mainSizer)
self.gs = None
self.features_panel.Hide()
self.content_panel.Hide()
self.features_text = ExpandoTextCtrl(self.features_panel,
size=(360, 290),
style=wx.TE_READONLY)
self.features_text.SetValue("Features listing")
self.Bind(EVT_ETC_LAYOUT_NEEDED, None, self.features_text)
self.features_sizer = wx.BoxSizer(wx.VERTICAL)
self.features_sizer.Add(self.features_text, 0, wx.CENTER, 5)
self.features_panel.SetSizer(self.features_sizer)
self.content_text = ExpandoTextCtrl(self.content_panel,
size=(360, 290),
style=wx.TE_READONLY)
self.content_text.SetValue("Script code")
self.Bind(EVT_ETC_LAYOUT_NEEDED, None, self.content_text)
self.content_sizer = wx.BoxSizer(wx.VERTICAL)
self.content_sizer.Add(self.content_text, 0, wx.CENTER, 5)
self.content_panel.SetSizer(self.content_sizer)
def on_analyze_press(self, event):
self.analyze()
def on_key_press(self, event):
keycode = event.GetKeyCode()
if keycode == wx.WXK_RETURN or keycode == wx.WXK_NUMPAD_ENTER:
self.analyze()
else:
event.Skip()
def analyze(self):
self.url = self.url_input.GetValue()
if not self.url:
return
try:
driver.get(self.url)
self.err_msg.SetLabel("")
except Exception as e:
self.err_msg.SetLabel(str(e))
print(e)
return
self.select_all_btn.Show()
# Uncomment to show diff button
# self.diff_btn.Show()
self.select_all_btn.SetValue(False)
self.features_panel.Show()
self.content_panel.Show()
self.features_text.SetValue("Features listing")
self.content_text.SetValue("Script code")
self.JavaScripts = {}
self.scriptButtons = []
self.choiceBoxes = []
while self.gs != None and len(self.gs.GetChildren()) > 0:
self.gs.Hide(self.number_of_buttons - 1)
self.gs.Remove(self.number_of_buttons - 1)
self.number_of_buttons -= 1
self.frame.fSizer.Layout()
html_source = driver.page_source
self.html = str(BeautifulSoup(html_source, 'html.parser'))
#Here is the part which extracts Scripts
scripts = driver.find_elements_by_tag_name("script")
numScripts = self.html.count("<script")
if numScripts % 2 != 0:
self.gs = wx.GridSizer(numScripts // 2 + 1, 4, 5, 5)
else:
self.gs = wx.GridSizer(numScripts // 2, 4, 5, 5)
cnt = 0
while "<script" in self.html:
sIndex = self.html.find("<script")
eIndex = self.html.find("</script>")
text = self.html[sIndex:eIndex + 9]
if ' src="' in text:
src = text.split(' src=')[1].split('"')[1].replace(
"http://", "").replace("https://", "")
src = src.split("?")[0]
contentText = ""
# Connect to the database.
conn = pymysql.connect(db=db_name,
user=db_user,
passwd=db_password,
host='localhost',
autocommit=True)
d = conn.cursor()
try:
sql = "SELECT filename FROM caching WHERE url LIKE '%{0}%'".format(
src)
d.execute(sql)
if d.rowcount > 0:
filename = d.fetchone()[0]
contentText = getScriptText(filename)
except:
contentText = ""
print("SCRIPT #", cnt)
print(text)
print(contentText[:500])
print("---" * 20)
d.close()
conn.close()
else:
contentText = text
self.html = self.html.replace(text,
"\n<!--script" + str(cnt) + "-->\n")
self.scriptButtons.append(
wx.ToggleButton(self.scripts_panel,
label="script" + str(cnt),
size=(100, 25)))
self.scriptButtons[cnt].Bind(wx.EVT_TOGGLEBUTTON,
self.on_script_press)
self.scriptButtons[cnt].myname = "script" + str(cnt)
self.gs.Add(self.scriptButtons[cnt], 0, wx.LEFT | wx.TOP, 25)
self.number_of_buttons += 1
choiceBox = wx.ComboBox(self.scripts_panel,
value="",
style=wx.CB_READONLY,
choices=("", "critical", "non-critical",
"translatable"))
choiceBox.Bind(wx.EVT_COMBOBOX, self.OnChoice)
choiceBox.index = len(self.choiceBoxes)
self.choiceBoxes.append(choiceBox)
self.gs.Add(choiceBox, 0, wx.TOP, 25)
self.number_of_buttons += 1
tmp = {}
for feature in features:
if feature in contentText:
tmp[feature] = contentText.count(feature)
tmp_sorted = OrderedDict(
sorted(tmp.items(), key=lambda x: x[1], reverse=True))
tmp = ""
for k, v in tmp_sorted.items():
tmp += "{0}: {1}\n".format(k, v)
self.JavaScripts["script" + str(cnt)] = [tmp, contentText, text]
cnt += 1
self.scripts_panel.SetSizer(self.gs)
self.frame.fSizer.Layout()
# check if we have already seen and saved the simplified page before in the DB
conn = pymysql.connect(db=db_name,
user=db_user,
passwd=db_password,
host='localhost',
autocommit=True)
d = conn.cursor()
mainName = driver.current_url
self.url = mainName
# if "https://" in mainName:
# mainName = "https://" + mainName[8:len(mainName)-1] + ":443/"
sql = "SELECT filename FROM caching WHERE url='{0}'".format(
mainName + "JScleaner.html")
d.execute(sql)
if d.rowcount > 0:
self.fileName = d.fetchone()[0]
else:
self.fileName = binascii.b2a_hex(os.urandom(15)).decode("utf-8")
while os.path.exists(PROXY_DATA_PATH + self.fileName):
self.fileName = binascii.b2a_hex(
os.urandom(15)).decode("utf-8")
sql = "INSERT INTO caching (url, filename) VALUES (%s,%s)"
d.execute(sql, (mainName + "JScleaner.html", self.fileName))
sql = "SELECT filename FROM caching WHERE url='{0}'".format(
mainName)
d.execute(sql)
print(mainName)
oldName = d.fetchone()[0]
shutil.copy(PROXY_DATA_PATH + oldName + ".h",
PROXY_DATA_PATH + self.fileName + ".h")
print("Encoding the JSCleaner version")
self.encode_save_index(self.html, self.fileName, PROXY_DATA_PATH)
d.close()
conn.close()
print("Loading the JScleaner version", self.url + "JScleaner.html")
driver.get(self.url + "JScleaner.html")
final_html = driver.execute_script(
"return document.getElementsByTagName('html')[0].innerHTML")
f = open("before.html", "w")
f.write(final_html)
f.close()
def on_all_press(self, event):
try:
toggle = event.GetEventObject().GetValue()
except:
toggle = True
if toggle:
# Insert all scripts
for name in self.JavaScripts:
if "<!--" + name + "-->" in self.html:
self.html = self.html.replace("<!--" + name + "-->",
self.JavaScripts[name][2])
# Toggle all script buttons
for btn in self.scriptButtons:
btn.SetValue(True)
else:
# Remove all scripts
for name in self.JavaScripts:
if self.JavaScripts[name][2] in self.html:
self.html = self.html.replace(self.JavaScripts[name][2],
"<!--" + name + "-->")
# Untoggle all script buttons
for btn in self.scriptButtons:
btn.SetValue(False)
self.encode_save_index(self.html, self.fileName, PROXY_DATA_PATH)
driver.get(self.url + "JScleaner.html")
def on_script_press(self, event):
try:
name = event.GetEventObject().myname
toggle = event.GetEventObject().GetValue()
except:
name = "script0"
toggle = True
if toggle:
JSContent = jsbeautifier.beautify(self.JavaScripts[name][1])
self.features_text.SetValue(name + "\n\n" +
self.JavaScripts[name][0])
self.content_text.SetValue(JSContent)
self.html = self.html.replace("<!--" + name + "-->",
self.JavaScripts[name][2])
else:
self.select_all_btn.SetValue(False)
self.features_text.SetValue("")
self.content_text.SetValue("")
self.html = self.html.replace(self.JavaScripts[name][2],
"<!--" + name + "-->")
self.encode_save_index(self.html, self.fileName, PROXY_DATA_PATH)
driver.get(self.url + "JScleaner.html")
def on_diff_press(self, event):
final_html = driver.execute_script(
"return document.getElementsByTagName('html')[0].innerHTML")
try:
f = open("after.html", "r")
before = f.read()
f.close()
f = open("before.html", "w")
f.write(before)
f.close()
except IOError:
pass
f = open("after.html", "w")
f.write(final_html)
f.close()
os.system(
"diff before.html after.html | sed '/<!--script/,/<\/script>/d'")
print("hello")
def encode_save_index(self, content, name, path):
with gzip.open(path + name + ".c", "wb") as f:
f.write(content.encode())
f.close
# print ("HTML is encoded and saved!")
content_size = os.path.getsize(path + name + ".c")
with open(path + name + ".h") as f:
new_text = ""
existing_size = ""
for line in f:
if "Content-Length:" in line:
existing_size = line.split(' ', 1)[1]
if (existing_size != ""):
with open(path + name + ".h") as f:
atext = f.read().replace(existing_size,
str(content_size) + "\n")
with open(path + name + ".h", "w") as f:
f.write(atext)
def OnChoice(self, event):
choiceBox = self.choiceBoxes[event.GetEventObject().index]
choiceBox.SetBackgroundColour(self.colors[choiceBox.GetValue()])
class TextLayerDialog(wx.Dialog):
"""
Controls setting options and displaying/hiding map overlay decorations
"""
def __init__(self,
parent,
ovlId,
title,
name='text',
pos=wx.DefaultPosition,
size=wx.DefaultSize,
style=wx.DEFAULT_DIALOG_STYLE):
wx.Dialog.__init__(self, parent, wx.ID_ANY, title, pos, size, style)
from wx.lib.expando import ExpandoTextCtrl, EVT_ETC_LAYOUT_NEEDED
self.ovlId = ovlId
self.parent = parent
if self.ovlId in self.parent.MapWindow.textdict.keys():
self.currText = self.parent.MapWindow.textdict[self.ovlId]['text']
self.currFont = self.parent.MapWindow.textdict[self.ovlId]['font']
self.currClr = self.parent.MapWindow.textdict[self.ovlId]['color']
self.currRot = self.parent.MapWindow.textdict[
self.ovlId]['rotation']
self.currCoords = self.parent.MapWindow.textdict[
self.ovlId]['coords']
self.currBB = self.parent.MapWindow.textdict[self.ovlId]['bbox']
else:
self.currClr = wx.BLACK
self.currText = ''
self.currFont = self.GetFont()
self.currRot = 0.0
self.currCoords = [10, 10]
self.currBB = wx.Rect()
self.sizer = wx.BoxSizer(wx.VERTICAL)
box = wx.GridBagSizer(vgap=5, hgap=5)
# show/hide
self.chkbox = wx.CheckBox(parent=self,
id=wx.ID_ANY,
label=_('Show text object'))
if self.parent.Map.GetOverlay(self.ovlId) is None:
self.chkbox.SetValue(True)
else:
self.chkbox.SetValue(
self.parent.MapWindow.overlays[self.ovlId]['layer'].IsActive())
box.Add(item=self.chkbox,
span=(1, 2),
flag=wx.ALIGN_LEFT | wx.ALL,
border=5,
pos=(0, 0))
# text entry
label = wx.StaticText(parent=self,
id=wx.ID_ANY,
label=_("Enter text:"))
box.Add(item=label, flag=wx.ALIGN_CENTER_VERTICAL, pos=(1, 0))
self.textentry = ExpandoTextCtrl(parent=self,
id=wx.ID_ANY,
value="",
size=(300, -1))
self.textentry.SetFont(self.currFont)
self.textentry.SetForegroundColour(self.currClr)
self.textentry.SetValue(self.currText)
# get rid of unneeded scrollbar when text box first opened
self.textentry.SetClientSize((300, -1))
box.Add(item=self.textentry, pos=(1, 1))
# rotation
label = wx.StaticText(parent=self, id=wx.ID_ANY, label=_("Rotation:"))
box.Add(item=label, flag=wx.ALIGN_CENTER_VERTICAL, pos=(2, 0))
self.rotation = wx.SpinCtrl(parent=self,
id=wx.ID_ANY,
value="",
pos=(30, 50),
size=(75, -1),
style=wx.SP_ARROW_KEYS)
self.rotation.SetRange(-360, 360)
self.rotation.SetValue(int(self.currRot))
box.Add(item=self.rotation, flag=wx.ALIGN_RIGHT, pos=(2, 1))
# font
fontbtn = wx.Button(parent=self, id=wx.ID_ANY, label=_("Set font"))
box.Add(item=fontbtn, flag=wx.ALIGN_RIGHT, pos=(3, 1))
self.sizer.Add(item=box, proportion=1, flag=wx.ALL, border=10)
# note
box = wx.BoxSizer(wx.HORIZONTAL)
label = wx.StaticText(
parent=self,
id=wx.ID_ANY,
label=_("Drag text with mouse in pointer mode "
"to position.\nDouble-click to change options"))
box.Add(item=label,
proportion=0,
flag=wx.ALIGN_CENTRE | wx.ALL,
border=5)
self.sizer.Add(item=box,
proportion=0,
flag=wx.EXPAND | wx.ALIGN_CENTER_VERTICAL
| wx.ALIGN_CENTER | wx.ALL,
border=5)
line = wx.StaticLine(parent=self,
id=wx.ID_ANY,
size=(20, -1),
style=wx.LI_HORIZONTAL)
self.sizer.Add(item=line,
proportion=0,
flag=wx.EXPAND | wx.ALIGN_CENTRE | wx.ALL,
border=5)
btnsizer = wx.StdDialogButtonSizer()
btn = wx.Button(parent=self, id=wx.ID_OK)
btn.SetDefault()
btnsizer.AddButton(btn)
btn = wx.Button(parent=self, id=wx.ID_CANCEL)
btnsizer.AddButton(btn)
btnsizer.Realize()
self.sizer.Add(item=btnsizer,
proportion=0,
flag=wx.EXPAND | wx.ALL | wx.ALIGN_CENTER,
border=5)
self.SetSizer(self.sizer)
self.sizer.Fit(self)
# bindings
self.Bind(EVT_ETC_LAYOUT_NEEDED, self.OnRefit, self.textentry)
self.Bind(wx.EVT_BUTTON, self.OnSelectFont, fontbtn)
self.Bind(wx.EVT_TEXT, self.OnText, self.textentry)
self.Bind(wx.EVT_SPINCTRL, self.OnRotation, self.rotation)
def OnRefit(self, event):
"""Resize text entry to match text"""
self.sizer.Fit(self)
def OnText(self, event):
"""Change text string"""
self.currText = event.GetString()
def OnRotation(self, event):
"""Change rotation"""
self.currRot = event.GetInt()
event.Skip()
def OnSelectFont(self, event):
"""Change font"""
data = wx.FontData()
data.EnableEffects(True)
data.SetColour(self.currClr) # set colour
data.SetInitialFont(self.currFont)
dlg = wx.FontDialog(self, data)
if dlg.ShowModal() == wx.ID_OK:
data = dlg.GetFontData()
self.currFont = data.GetChosenFont()
self.currClr = data.GetColour()
self.textentry.SetFont(self.currFont)
self.textentry.SetForegroundColour(self.currClr)
self.Layout()
dlg.Destroy()
def GetValues(self):
"""Get text properties"""
return {
'text': self.currText,
'font': self.currFont,
'color': self.currClr,
'rotation': self.currRot,
'coords': self.currCoords,
'active': self.chkbox.IsChecked()
}
class MyPanel(wx.Panel):
"""The GUI of the tool."""
def __init__(self, parent):
wx.Panel.__init__(self, parent)
self.frame = parent
# start Chrome webdriver
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--proxy-server=%s' % PROXY)
# chrome_options.add_argument('--auto-open-devtools-for-tabs')
caps = DesiredCapabilities.CHROME
caps['goog:loggingPrefs'] = {'performance': 'INFO'}
chrome_options.add_experimental_option('perfLoggingPrefs',
{'enablePage': True})
self.driver = webdriver.Chrome(options=chrome_options,
desired_capabilities=caps)
self.driver.execute_cdp_cmd('Network.enable', {})
self.driver.execute_cdp_cmd('Network.setCacheDisabled',
{'cacheDisabled': True})
self.driver.set_window_size(650, 750)
self.driver.set_window_position(0, 0)
self.main_sizer = wx.BoxSizer(wx.VERTICAL)
original_options = webdriver.ChromeOptions()
original_options.add_argument('--proxy-server=' + REMOTE_PROXY_IP +
':8082')
self.original = webdriver.Chrome(options=original_options)
self.original.set_window_size(650, 750)
self.original.set_window_position(650, 0)
# TextCtrl for user to input URL of site to analyze
self.url_input = wx.TextCtrl(self, style=wx.TE_LEFT)
self.url_input.SetValue("http://yasirzaki.net/")
self.url_input.Bind(wx.EVT_KEY_DOWN, self.on_key_press)
self.main_sizer.Add(self.url_input,
flag=wx.EXPAND | wx.TOP | wx.LEFT | wx.RIGHT,
border=25)
# StaticText field for error messages
self.err_msg = wx.StaticText(self, label="")
self.main_sizer.Add(self.err_msg, flag=wx.LEFT, border=25)
analyze_btn = wx.Button(self, label='Analyze page')
analyze_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.main_sizer.Add(analyze_btn, flag=wx.ALL | wx.CENTER, border=5)
hbox = wx.BoxSizer(wx.HORIZONTAL)
self.scripts_panel = ScrolledPanel(self, size=(375, 550))
self.scripts_panel.SetupScrolling()
hbox.Add(self.scripts_panel)
self.content_panel = ScrolledPanel(self, size=(375, 550))
self.content_panel.SetupScrolling()
hbox.Add(self.content_panel, flag=wx.CENTER, border=5)
self.main_sizer.Add(hbox, flag=wx.CENTER | wx.BOTTOM, border=25)
hbox = wx.BoxSizer(wx.HORIZONTAL)
self.apply_btn = wx.Button(self, label='Apply Selection')
self.apply_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.apply_btn.SetToolTip('Preview changes in the browser window.')
self.apply_btn.Hide()
hbox.Add(self.apply_btn, border=5)
self.save_btn = wx.Button(self, label='Save and load simplified page')
self.save_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.save_btn.SetToolTip(
'Save changes in new folder and push to the remote proxy.')
self.save_btn.Hide()
hbox.Add(self.save_btn, border=5)
self.diff_btn = wx.Button(self, label='Get diff')
self.diff_btn.Bind(wx.EVT_BUTTON, self.on_button_press)
self.diff_btn.SetToolTip(
'Print diff before and after changes to terminal window.')
self.diff_btn.Hide()
hbox.Add(self.diff_btn, border=5)
self.main_sizer.Add(hbox, flag=wx.BOTTOM | wx.CENTER, border=25)
self.SetSizer(self.main_sizer)
self.url = self.url_input.GetValue()
self.suffix = "?JSTool=none"
self.script_sizer = wx.BoxSizer(wx.VERTICAL)
self.script_buttons = []
self.choice_boxes = []
self.number_of_buttons = 0
self.blocked_urls = []
self.content_panel.Hide()
self.content_text = ExpandoTextCtrl(self.content_panel,
size=(375, 275),
style=wx.TE_READONLY)
self.content_text.SetValue("Script code")
self.Bind(EVT_ETC_LAYOUT_NEEDED, None, self.content_text)
self.content_sizer = wx.BoxSizer(wx.VERTICAL)
self.content_sizer.Add(self.content_text, flag=wx.CENTER)
self.content_panel.SetSizer(self.content_sizer)
self.script_tree = AnyNode(id='root')
self.images = {}
self.yasir = {}
def on_button_press(self, event):
"""Handle wx.Button press."""
btn = event.GetEventObject()
if btn.GetLabel() == 'Analyze page':
self.analyze()
elif btn == self.diff_btn:
self.on_diff_press()
elif btn == self.apply_btn:
self.on_apply_press()
elif btn == self.save_btn:
self.on_save()
def on_key_press(self, event):
"""Handle keyboard input."""
keycode = event.GetKeyCode()
if keycode == wx.WXK_RETURN or keycode == wx.WXK_NUMPAD_ENTER:
self.analyze()
else:
event.Skip()
def add_button(self, script, index, depth, vector): # copies
"""Add script to self.script_buttons at index and update display."""
hbox = wx.BoxSizer(wx.HORIZONTAL)
hbox.AddSpacer(depth * 25)
# Create button
# if copies > 1: do something to differentiate it
self.script_buttons.insert(
index,
wx.CheckBox(self.scripts_panel, label=script.split("/")[-1][:9]))
self.script_buttons[index].myname = script
self.script_buttons[index].Bind(wx.EVT_CHECKBOX, self.on_script_press)
self.script_buttons[index].SetToolTip(script)
hbox.Add(self.script_buttons[index], flag=wx.ALL, border=5)
self.number_of_buttons += 1
# Create combobox
# choice_box = wx.ComboBox(self.scripts_panel, value="", style=wx.CB_READONLY, choices=(
# "", "critical", "non-critical", "replaceable"))
# choice_box.Bind(wx.EVT_COMBOBOX, self.on_choice)
# choice_box.index = len(self.choice_boxes)
# self.choice_boxes.insert(index, choice_box)
# hbox.Add(choice_box, flag=wx.ALL, border=5)
# self.number_of_buttons += 1
# Add labels
if script[:6] != 'script' and vector is not None:
category = ML_MODEL.predict([pandas.Series(vector)]).item(0)
confidence = np.amax(
ML_MODEL.predict_proba([pandas.Series(vector)]))
self.script_buttons[index].category = category
self.script_buttons[index].confidence = confidence
text = str(category) + ": " + str(int(confidence * 100)) + "%"
label = wx.StaticText(self.scripts_panel,
label=text,
style=wx.BORDER_RAISED)
label.SetBackgroundColour(tuple(CATEGORIES[category]['color']))
tool_tip = CATEGORIES[category]['description']
label.SetToolTip(tool_tip)
self.script_buttons[index].label = label
hbox.Add(label, flag=wx.ALL, border=5)
self.yasir[script] = self.script_buttons[index]
self.script_sizer.Insert(index, hbox)
self.frame.frame_sizer.Layout()
def format_src(self, src: str):
"""Return formatted src string to be requested."""
if src[:4] != "http":
if src[0] == "/":
if src[1] == "/":
src = "https:" + src
else:
src = self.url + src[1:]
else:
src = self.url + src
return src
def block_all_scripts(self):
"""Adds all scripts in self.script_tree to self.blocked_urls."""
self.blocked_urls.clear()
for node in PreOrderIter(self.script_tree):
if node.id[:6] != "script" and not node.is_root:
self.blocked_urls.append(node.id)
def wait_for_load(self):
"""Wait for page source to stop changing."""
html = self.driver.page_source
time.sleep(WAIT_LOAD_TIME)
while html != self.driver.page_source:
html = self.driver.page_source
time.sleep(WAIT_LOAD_TIME)
def analyze(self):
"""Do everything."""
def reset_display():
# Reset display
self.suffix = "?JSTool=none"
self.script_buttons.clear()
self.choice_boxes.clear()
self.number_of_buttons = 0
# self.diff_btn.Show()
self.apply_btn.Show()
self.save_btn.Show()
self.content_panel.Show()
self.content_text.SetValue("Script code")
while self.script_sizer.GetChildren():
self.script_sizer.Hide(0)
self.script_sizer.Remove(0)
self.images.clear()
def get_index_html():
# Get index.html from remote proxy
return get_resource(self.url)
def parse_html(html: str):
# Add index.html scripts to self.script_tree
cnt = 1
if not html:
return
while "<script" in html:
src = ""
script_name = "script" + str(cnt)
start_index = html.find("<script")
end_index = html.find("</script>")
text = html[start_index:end_index + 9]
new_node = AnyNode(id=script_name,
parent=self.script_tree,
content=text,
vector=extract_features(text),
count=1)
if ' src="' in text: # BeautifulSoup turns all single quotes into double quotes
src = text.split(' src="')[1].split('"')[0]
src = self.format_src(src)
try:
node = anytree.cachedsearch.find(
self.script_tree, lambda node: node.id == src)
except anytree.search.CountError:
logging.warning(
'multiple possible parents: more than one node with id = %s',
src)
if node:
node.parent = new_node
html = html.replace(text, "\n<!--" + script_name + "-->\n")
cnt += 1
def create_buttons():
# Add checkboxes to display
# Check all
self.add_button('Check all', 0, 1, None)
index = 1
# All other script checkboxes
for node in PreOrderIter(self.script_tree):
if node.is_root:
continue
node.button = index
# vector = extract_features(node.content)
self.add_button(node.id, index, node.depth,
get_attribute(node, 'vector')) # node.count
checkbox = self.script_buttons[index]
if (get_attribute(checkbox, 'confidence') is not None
and get_attribute(
checkbox, 'confidence') < CONFIDENCE_THRESHOLD):
# run clustering if confidence less than threshold
checkbox.category = CLUSTER.predict(script=str(
node.content),
preprocess=True)
label = get_attribute(checkbox, 'label')
if label:
label.SetLabel(checkbox.category)
label.SetBackgroundColour(
tuple(CATEGORIES[checkbox.category]['color']))
label.SetToolTip(
CATEGORIES[checkbox.category]['description'])
if get_attribute(checkbox,
'category') not in BLOCKED_CATEGORIES:
# ads / marketing scripts disabled by default
try:
if node.id[:6] != "script":
self.blocked_urls.remove(node.id)
except ValueError:
logging.debug("Could not remove %s from blocked urls",
node.id)
self.check_boxes(True, node)
index += 1
self.scripts_panel.SetSizer(self.script_sizer)
self.frame.frame_sizer.Layout()
def functional_dependency():
# functional dependencies?
try:
tmp_dep = perf.get_dependency(self.url)
# tmp_dep = [['https://ws.sharethis.com/button/async-buttons.js', 'https://www.google-analytics.com/analytics.js', 'https://ws.sharethis.com/button/buttons.js'], ['https://www.googletagmanager.com/gtm.js?id=GTM-WBDQQ5', 'https://www.googleadservices.com/pagead/conversion_async.js'], ['https://www.unicef.org/sites/default/files/js/js_B7pS3ddmNLFYOJi3j28odiodelMu-EhaOeKlHZ8E6y0.js', 'https://www.unicef.org/themes/custom/unicef/assets/src/js/init-blazy.js?v=1.x', 'https://www.unicef.org/sites/default/files/js/js_dWWS6YNlsZWmXLboSy3PIiSD_Yg3sRxwjbMb52mdNyw.js', 'https://www.unicef.org/sites/default/files/js/js_cLlwgRdoiVfjtFxLqlXX-aVbv3xxfX_uMCsn7iJqNpA.js']]
print("\n\n-------- DEPENDENCY LABELS CHANGED --------")
mapping = {'non-critical': 0, 'translatable': 1, 'critical': 2}
mapping2 = {
0: 'non-critical',
1: 'translatable',
2: 'critical'
}
for a in tmp_dep:
tmp_label = 0
for i in a:
if i not in self.yasir or self.yasir[
i].category not in mapping:
continue
if mapping[self.yasir[i].category] > tmp_label:
tmp_label = mapping[self.yasir[i].category]
for i in a:
if i not in self.yasir or self.yasir[
i].category not in mapping:
continue
if self.yasir[i].category != mapping2[tmp_label]:
print("****", i, mapping2[tmp_label],
self.yasir[i].category)
print("\n\n")
except RuntimeError:
pass
def display_loading_message():
# Never managed to get this part to display before spinning wheel of death
self.err_msg.SetForegroundColour((0, 0, 0))
self.err_msg.SetLabel("Loading page... please wait")
self.Update()
def similarity():
# Print script pairs in self.script_tree with Jaccard similarity > SIMILARITY_THRESHOLD
names = []
scripts = []
for node in PreOrderIter(self.script_tree):
if node.is_root:
continue
names.append(node.id)
scripts.append(str(node.content))
results = similarity_comparison(scripts, SIMILARITY_THRESHOLD)
if results:
print("---" * 20)
print('scripts with similarity > %.2f' % SIMILARITY_THRESHOLD)
for tup in results:
print('%s %s %.2f' % (names[tup[0]], names[tup[1]], tup[2]))
def compare_image_sizes(images):
# Print difference in original and rendered image sizes for image URLs in images
for url in images:
if url[:4] == 'data':
# URI rather than URL
url = url.partition(';')[-1]
body = url.partition(',')[-1]
if url[:6] == 'base64':
body = base64.b64decode(body)
else:
body = get_resource(url)
try:
stream = BytesIO(body)
except TypeError:
logging.warning("body in %s, not in bytes", type(body))
stream = BytesIO(body.encode(ENCODING))
try:
width, height = get_image_size_from_bytesio(
stream, DEFAULT_BUFFER_SIZE)
self.images[url] = {}
self.images[url]['ow'] = width
self.images[url]['oh'] = height
except UnknownImageFormat as error:
logging.exception(str(error))
except struct.error as error:
logging.error(str(error))
for img in self.driver.find_elements_by_tag_name('img'):
url = img.get_attribute('src')
if url not in self.images.keys():
self.images[url] = {}
self.images[url]['rw'] = img.size['width']
self.images[url]['rh'] = img.size['height']
logging.info("---" * 20)
logging.info("potential image improvements:")
for url, dimensions in self.images.items():
if len(dimensions.keys()) == 4:
# Successfully parsed original and rendered dimensions
logging.info(url)
logging.info("original: %d x %d", dimensions['ow'],
dimensions['oh'])
logging.info("rendered: %d x %d", dimensions['rw'],
dimensions['rh'])
display_loading_message()
# Reset values
self.url = self.url_input.GetValue()
if self.url[-1] != "/":
self.url = self.url + "/"
if not self.url:
return
reset_display()
self.script_tree = AnyNode(id=self.url)
try:
file_path = PATH + "/reports/" + self.url.split("/")[2]
if not os.path.exists(file_path):
os.mkdir(file_path)
with open(file_path + "/script_tree.txt", 'r') as f:
logging.debug('importing script tree...')
importer = JsonImporter()
self.script_tree = importer.read(f)
with open(file_path + "/images.json", 'r') as f:
images = json.load(f)
except FileNotFoundError:
logging.debug('script tree does not yet exist, building now')
# Get original page and parse external scripts
self.driver.execute_cdp_cmd('Network.setBlockedURLs', {'urls': []})
epoch_in_milliseconds = time.time() * 1000
try:
self.driver.get(self.url)
self.err_msg.SetLabel("")
except InvalidArgumentException as exception:
self.err_msg.SetForegroundColour((255, 0, 0)) # make text red
self.err_msg.SetLabel(str(exception))
return
self.wait_for_load()
self.script_tree = AnyNode(id=self.url)
scripts, images = self.parse_log(epoch_in_milliseconds)
for script in scripts:
# pylint: disable=undefined-loop-variable
# pylint: disable=cell-var-from-loop
parent = anytree.cachedsearch.find(
self.script_tree,
lambda node: node.id == self.format_src(script['parent']))
# Check if this node already exists
node = anytree.cachedsearch.find(
self.script_tree,
lambda node: node.id == self.format_src(script['url']))
if node and node.parent == parent:
logging.warning('duplicate script! %s',
self.format_src(script['url']))
node.count += 1
else:
AnyNode(id=self.format_src(script['url']),
parent=parent,
content=script['content'],
vector=extract_features(script['content']),
count=1)
# Check image differences
compare_image_sizes(images)
# Parse inline scripts
html = get_index_html()
parse_html(html)
# self.print_scripts()
# Export script tree
logging.debug('exporting script tree...')
exporter = JsonExporter()
with open(
PATH + "/reports/" + self.url.split("/")[2] +
"/script_tree.json", "w") as f:
exporter.write(self.script_tree, f)
logging.debug('done')
# Export images
with open(
PATH + "/reports/" + self.url.split("/")[2] +
"/images.json", "w") as f:
json.dump(images, f)
# Check similarity
# similarity()
# Create buttons
self.block_all_scripts()
create_buttons()
# Print functional dependencies
# functional_dependency()
# Get page with all scripts removed
self.on_apply_press()
try:
self.original.get(self.url)
except InvalidArgumentException as e:
logging.error(e.what())
# Used for diff
# final_html = BeautifulSoup(self.driver.execute_script(
# "return document.getElementsByTagName('html')[0].innerHTML"), 'html.parser')
# file_stream = open("before.html", "w")
# file_stream.write(final_html.prettify())
# file_stream.close()
def on_check_all(self, toggle):
"""Handle 'Select All' checkbox toggle."""
self.suffix = "?JSTool="
for btn in self.script_buttons:
btn.SetValue(toggle)
if toggle and btn.myname[:6] == "script":
self.suffix += "_" + btn.myname[6:]
if toggle:
# Toggle all script buttons
self.blocked_urls.clear()
else:
# Untoggle all script buttons
self.block_all_scripts()
self.suffix += "none"
def on_apply_press(self):
"""Send request for page with changes."""
self.driver.execute_cdp_cmd('Network.setBlockedURLs',
{'urls': self.blocked_urls})
self.suffix = "?JSTool="
for btn in self.script_buttons:
if btn.GetValue() and btn.myname[:6] == "script":
self.suffix += "_" + btn.myname[6:]
if self.suffix == "?JSTool=":
self.suffix += "none"
self.driver.get(self.url + self.suffix)
self.err_msg.SetLabel("")
def on_script_press(self, event):
"""Handle script button press."""
name = event.GetEventObject().myname
toggle = event.GetEventObject().GetValue()
if name == 'Check all':
self.on_check_all(toggle)
return
node = anytree.cachedsearch.find(self.script_tree,
lambda node: node.id == name)
if not get_attribute(node, 'content'):
node.content = get_resource(node.id)
self.content_text.SetValue(name + "\n\n" + str(node.content))
self.check_boxes(toggle, node)
def check_boxes(self, toggle: bool, node: AnyNode):
"""
Check (toggle = true) or uncheck (toggle = false) boxes
corresponding to node while keeping dependencies intact.
All ancestors of node are also checked if node is checked,
and all children of node are also unchecked if node is unchecked.
"""
if toggle:
while node.depth > 1:
self.script_buttons[node.button].SetValue(True)
try:
self.blocked_urls.remove(node.id)
except ValueError:
logging.debug("Could not remove %s from blocked urls",
node.id)
node = node.parent
self.script_buttons[node.button].SetValue(True)
if node.id[:6] != "script":
try:
self.blocked_urls.remove(node.id)
except ValueError:
logging.debug("Could not remove %s from blocked urls",
node.id)
else:
for descendant in node.descendants:
self.script_buttons[descendant.button].SetValue(False)
self.blocked_urls.append(descendant.id)
if node.id[:6] != "script":
self.blocked_urls.append(node.id)
def on_diff_press(self):
"""Print diff to terminal."""
after = BeautifulSoup(
self.driver.execute_script(
"return document.getElementsByTagName('html')[0].innerHTML"),
'html.parser')
try:
file_stream = open("after.html", "r")
before = file_stream.read()
file_stream.close()
file_stream = open("before.html", "w")
file_stream.write(before)
file_stream.close()
before = BeautifulSoup(before, 'html.parser')
except IOError:
pass
file_stream = open("after.html", "w")
file_stream.write(after.prettify())
file_stream.close()
os.system(r"git diff --no-index before.html after.html")
# os.system(r"diff before.html after.html | sed '/<!--script/,/<\/script>/d'")
def on_save(self):
"""Generate report and save in reports folder."""
if not os.path.exists(PATH + "/reports"):
os.mkdir(PATH + "/reports")
file_path = PATH + "/reports/" + self.url.split("/")[2]
if not os.path.exists(file_path):
os.mkdir(file_path)
logging.info("Writing script files...")
critical = open(file_path + '/critical.txt', 'w')
noncritical = open(file_path + '/noncritical.txt', 'w')
webalmanac = open(file_path + '/webalmanac.txt', 'w')
# labels = open(PATH + "/reports/labels.csv", 'a')
for node in PreOrderIter(self.script_tree):
if node.is_root or node.id[:6] == 'script':
continue
checkbox = self.script_buttons[get_attribute(node, 'button')]
if checkbox.GetValue():
critical.write(node.id + "\n")
else:
noncritical.write(node.id + "\n")
label = get_attribute(checkbox, 'label')
if label and label.GetLabel() != 'critical' and label.GetLabel(
) != 'non-critical':
webalmanac.write(node.id + "\n")
webalmanac.write(label.GetLabel() + "\n")
# if checkbox.GetValue():
# labels.write(str(node.vector.to_list()) + "," +
# labels.write(str(node.vector) + "," +
# label.GetLabel() + ",critical\n")
# else:
# labels.write(str(node.vector.to_list()) + "," +
# labels.write(str(node.vector) + "," +
# label.GetLabel() + ",non-critical\n")
critical.close()
noncritical.close()
webalmanac.close()
# labels.close()
logging.info("Writing index file...")
index = open(file_path + '/index.html', 'w')
# pickle.dump(get_resource(self.url + self.suffix) + "\n", index)
index.write(get_resource(self.url + self.suffix))
index.close()
logging.info("Writing images file...")
images = open(file_path + '/images.txt', 'w')
for url, dimensions in self.images.items():
images.write(url + "\n")
if 'ow' in dimensions and 'oh' in dimensions:
images.write("original: %d x %d\n" %
(dimensions['ow'], dimensions['oh']))
if 'rw' in dimensions and 'rh' in dimensions:
images.write("rendered: %d x %d\n" %
(dimensions['rw'], dimensions['rh']))
images.close()
self.err_msg.SetForegroundColour((0, 0, 0))
logging.info("Report generated in %s", file_path)
# Send report to proxy
multipart_form_data = {
'html_content': (open(file_path + '/index.html', 'rb')),
'blocked_URLs': (open(file_path + '/noncritical.txt', 'rb')),
'images': (open(file_path + '/images.txt', 'rb')),
'url': self.url,
}
response = requests.post("http://" + REMOTE_PROXY_IP +
":9000/JSCleaner/JSAnalyzer.py",
files=multipart_form_data)
if response.status_code == 200:
self.err_msg.SetLabel("Report sent to external proxy")
else:
self.err_msg.SetLabel(
"Report could not be sent - report generated in %s" %
file_path)
logging.error(response.status_code)
logging.error(response.headers)
logging.debug(response.text)
# Load simplified page from proxy
simplified_options = webdriver.ChromeOptions()
simplified_options.add_argument('--proxy-server=' + REMOTE_PROXY_IP +
':8083')
simplified = webdriver.Chrome(options=simplified_options)
simplified.set_window_size(600, 750)
simplified.set_window_position(650, 0)
simplified.get(self.url + '?JSCleaner')
input()
simplified.close()
def on_choice(self, event):
"""Handle choiceBox selection."""
choice_box = self.choice_boxes[event.GetEventObject().index]
colors = {
"": wx.Colour(255, 255, 255, 100),
"critical": wx.Colour(255, 0, 0, 100),
"non-critical": wx.Colour(0, 255, 0, 100),
"translatable": wx.Colour(0, 0, 255, 100)
}
choice_box.SetBackgroundColour(colors[choice_box.GetValue()])
def parse_log(self, epoch_in_milliseconds):
"""Return list of scripts requested since epoch_in_milliseconds."""
scripts = []
images = []
log = self.driver.get_log('performance')
log = log[bisect.bisect_left([entry['timestamp']
for entry in log], epoch_in_milliseconds
):]
log = [json.loads(entry['message'])['message'] for entry in log]
def is_script_request(message):
if message['method'] == 'Network.requestWillBeSent':
if message['params']['type'] == 'Script':
return True
return False
def is_image_request(message):
if message['method'] == 'Network.requestWillBeSent':
if message['params']['type'] == 'Image':
return True
return False
# def is_script_response(message):
# if message['method'] == 'Network.responseReceived':
# if 'javascript' in message['params']['response']['mimeType']:
# return True
# return False
def is_data_received(message):
if message['method'] == 'Network.dataReceived':
return True
return False
def get_request_info(message):
request_id = message['params']['requestId']
request_url = message['params']['request']['url']
initiator = message['params']['initiator']
if initiator['type'] == 'parser':
# from index.html as src, need to identify script number somehow...
# there are line numbers but are those usable?
initiator = initiator['url']
elif initiator['type'] == 'script':
# pick last thing in callFrames because first thing doesn't always have URL?
# need better understanding
# each script has its own ID... if only I could figure out how to use it
if initiator['stack']['callFrames']:
initiator = initiator['stack']['callFrames'][-1]['url']
return [request_id, request_url, initiator]
script_requests = []
# script_responses = []
image_requests = []
data_received = []
for message in log:
if is_script_request(message):
script_requests.append(message)
# elif is_script_response(message):
# script_responses.append(message['params']['requestId'])
elif is_image_request(message):
image_requests.append(message)
elif is_data_received(message):
data_received.append(message['params']['requestId'])
for request in script_requests:
request_id, url, initiator = get_request_info(request)
if request_id in data_received:
content = get_resource(url)
scripts.append({
'url': url,
'parent': initiator,
'content': content
})
for request in image_requests:
request_id, url, initiator = get_request_info(request)
if request_id in data_received:
images.append(url)
return (scripts, images)
def print_scripts(self):
"""Print script tree."""
print(RenderTree(self.script_tree).by_attr('id'))
print("---" * 20)
def print_blocked_scripts(self):
"""Print blocked URLs."""
print('BLOCKED SCRIPTS:')
for url in self.blocked_urls:
print("\t", url)
