def qvariant_encode(value):
buffer = QBuffer()
buffer.open(QIODevice.ReadWrite)
stream = QDataStream(buffer)
stream.writeQVariant(value)
buffer.close()
return buffer.buffer().toBase64().data().strip()
def qvariant_encode(value):
buffer = QBuffer()
buffer.open(QIODevice.ReadWrite)
stream = QDataStream(buffer)
stream.writeQVariant(value)
buffer.close()
return buffer.buffer().toBase64().data().strip()
def set_clipboard_image(self):
"""Export the formatted output to an image and store it in the
clipboard.
The image stored in the clipboard is a PNG file with alpha
transparency.
"""
div = self.view.page().mainFrame().findFirstElement('.output')
images = {}
for background in (Qt.transparent, Qt.white):
image = QImage(div.geometry().size(),
QImage.Format_ARGB32_Premultiplied)
image.fill(background)
painter = QPainter(image)
div.render(painter)
painter.end()
images[background] = image
# Windows needs this buffer hack to get alpha transparency in
# the copied PNG.
buffer_ = QBuffer()
buffer_.open(QIODevice.WriteOnly)
images[Qt.transparent].save(buffer_, 'PNG')
buffer_.close()
data = QMimeData()
data.setData('PNG', buffer_.data())
data.setImageData(images[Qt.white])
QApplication.clipboard().setMimeData(data)
def qvariant_decode(data):
byteArray = QByteArray.fromBase64(data)
buffer = QBuffer(byteArray)
buffer.open(QIODevice.ReadOnly)
stream = QDataStream(buffer)
result = stream.readQVariant()
buffer.close()
return result
def qvariant_decode(data):
byteArray = QByteArray.fromBase64(data)
buffer = QBuffer(byteArray)
buffer.open(QIODevice.ReadOnly)
stream = QDataStream(buffer)
result = stream.readQVariant()
buffer.close()
return result
def mapBytes(self, kode):
im = self.draw(kode)
if isinstance(kode, UKode):
qxm = pilfunc.pixmap2qt(im)
elif isinstance(kode, (BKode, SKode)):
qxm = pilfunc.bitmap2qt(im)
else:
raise ValueError
bts = QByteArray()
buffer = QBuffer(bts)
buffer.open(QIODevice.WriteOnly)
qxm.save(buffer, "png")
buffer.close()
return bts
def svgImage(svgfile, file=None):
qs = QSvgWidget()
qs.load(svgfile)
qim = QPixmap.grabWidget(qs)
bts = QByteArray()
buffer = QBuffer(bts)
buffer.open(QIODevice.WriteOnly)
qim.save(buffer, "png")
bts = buffer.data().data()
buffer.close()
if type(file) == str:
assert os.path.splitext(file)[1].lower() == ".png"
with open(file, "bw") as f:
f.write(bts)
elif hasattr(file, 'write'):
file.write(bts)
else:
return bts
def main(url, output, option={}):
result = {
"error":[],
"page":{},
"resources":[],
"capture":None,
}
#savedir = appdir + "/artifacts/ghost/" + output
#dump = savedir + "/ghost.pkl"
savedir = os.path.join(appdir, "artifacts/ghost")
dump = savedir + "/" + output
try:
#if os.path.exists(savedir):
# shutil.rmtree(savedir)
#os.makedirs(savedir)
with open(dump, 'wb') as d:
#umsgpack.dump(result, d)
json.dump(result, d)
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
defaults = {
"wait_timeout":60,
"display": False,
"viewport_size": (800, 600),
"plugins_enabled": True,
"java_enabled": True,
}
proxy_url = None
http_method = "GET"
req_headers = None
body = None
if option:
if "user_agent" in option:
defaults["user_agent"] = str(option["user_agent"])
if "timeout" in option:
defaults["wait_timeout"] = int(option["timeout"])
if "proxy" in option:
proxy_url = option["proxy"]
if "method" in option:
http_method = option["method"]
if "headers" in option:
req_headers = option["headers"]
if "post_data" in option:
body = str(option["post_data"])
logger.info(defaults)
ghost = None
try:
ghost = Ghost(
#log_level=logging.DEBUG,
log_level=logging.INFO,
plugin_path=[ appdir + '/plugins', '/usr/lib/mozilla/plugins', ],
defaults=defaults,
)
except Exception as e:
logger.error("ghost init failed. " + str(e))
result["error"].append(str(e))
with ghost.start() as session:
if proxy_url:
try:
type = proxy_url.split(":")[0]
server = proxy_url.split("/")[2]
host = server.split(":")[0]
port = server.split(":")[1]
session.set_proxy(str(type),host=str(host),port=int(port),)
except Exception as e:
logger.debug(e)
headers = {}
if req_headers:
logger.debug(req_headers)
for h in req_headers:
headers[str(h)] = str(req_headers[h])
logger.debug(headers)
if hasattr(ghost, "xvfb"):
logger.info(ghost.xvfb)
page = None
resources =None
try:
page, resources = session.open(
url.decode("utf-8"),
method = http_method,
headers = headers,
body = body
)
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
#if error:
# result["error"] = error.spilt(".")[-1]
if page:
result["page"] = {
"url":page.url,
"http_status":page.http_status,
"headers":page.headers,
#"content":session.content.encode("utf-8"),
"content":base64.b64encode(session.content.encode("utf-8")),
"seq":0,
#"error":page.error.encode("utf-8").split(".")[-1],
"error":page.error.split(".")[-1],
}
try:
image = session.capture()
ba = QByteArray()
buffer = QBuffer(ba)
buffer.open(QIODevice.WriteOnly)
image.save(buffer, "PNG")
bio = BytesIO(ba)
bio.seek(0)
#result["capture"] = bio.read()
result["capture"] = base64.b64encode(bio.read())
bio.flush()
bio.close()
ba.clear()
buffer.close()
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
if resources:
seq = 0
for r in resources:
seq += 1
#logger.debug(r.url)
dict = {
"url":r.url,
"http_status":r.http_status,
"headers":r.headers,
#"content":r.content.encode("utf-8"),
"content":base64.b64encode(r.content),
"seq":seq,
#"error":r.error.encode("utf-8").split(".")[-1],
"error":r.error.split(".")[-1],
}
result["resources"].append(dict)
with open(dump, 'wb') as d:
#umsgpack.dump(result, d)
json.dump(result, d)
logger.debug(dump)
ghost.exit()
return dump
def runparser(filelistdirs,logdir):
print 'Start running the Translator'
#os.chdir(logdir)
filelistdirs=filter(None,filelistdirs)
print 'D1',filelistdirs
for dir in xrange(len(filelistdirs)):
curdir=filelistdirs[dir]
print 'D2',curdir
newdir=os.path.join(logdir,os.path.basename(curdir)).replace('\\','/')
if not os.path.exists(newdir):
os.mkdir(newdir)
filelist = os.listdir(curdir)
print filelist
os.chdir(curdir)
for z in xrange(len(filelist)):
### Start the xml parser
tree = ET.parse(filelist[z])
root = tree.getroot()
'''create a html result file '''
filename= os.path.basename(filelist[z])
logfile=os.path.join(newdir,filename.replace('.onb','.html')).replace('\\','/')
print logfile
f=open(logfile,'w')
headers='''
<!doctype html>
<head>
<title>OMWEBbook</title>
<meta charset="utf-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<script src="../jquery.min1.10.2.js"></script>
<script src='../dygraph-combined.js'></script>
<link rel="stylesheet" href="../bootstrap.min.css">
<script src="../bootstrap.min.js"></script>
<link rel="stylesheet" href="../codemirror.css">
<script src="../codemirror.js"></script>
<script src="../modelica.js"></script>
<link rel="stylesheet" href="../custom.css">
<script src="../autorefresh.js"></script>
<script src="../evalnotebook.js"></script>
</head>
<body>
<div class="navbar navbar-default navbar-fixed-top">
<div class="container-fluid">
<div class="navbar-header">
<a class="navbar-brand" href="#">OMWebBook</a>
</div>
<button id="evaluate" type="button" class="btn btn-success navbar-btn">Evaluate Cell</button>
<button id="evaluateall" type="button" class="btn btn-success navbar-btn">Eval All</button>
<img id="progressbar" src="../ajax-loader.gif" class="img-rounded" alt="Cinque Terre">
</div>
</div>
<div class="container">
<br> <br> <br>
'''
f.write(headers)
count=0
check=0
sectioncount=1
subsectioncount=0.01
sectioncheck=False
subsectioncheck=False
currentlevel=''
g=1
g1=1
for node in tree.iter():
if (node.tag=='TextCell'):
imagelist=node.findall('Image')
html=node.find('Text').text
soup = BeautifulSoup(html)
for a in soup.findAll('a'):
staticlink="".join(['static','/',a['href']]).replace('.onb','.html')
a['href']=a['href'].replace(a['href'], staticlink)
#print staticlink
findp=[]
for p in soup.findAll('p'):
checkempty=p['style'].replace(' ','').split(";")
val="-qt-paragraph-type:empty" in checkempty
if (val==False):
# if (p.find('img')==None):
# findp.append(p)
# else:
findp.append(p)
for i in xrange(len(findp)):
#x=findp[i].text
x=findp[i]
if(x.find('img')is None):
if(x!=''):
if(node.attrib['style']=='Text'):
#partext='\n'.join(['<p align="justify" contenteditable=False>',str(x),'</p>'])
#x["align"]='justify'
x=re.sub(r'[^\x00-\x7F]+',' ', str(x))
partext='\n'.join([str(x)])
f.write(partext)
f.write('<br>')
f.write('\n')
elif(node.attrib['style']=='Title'):
print 'title'
t=findp[i].text
t=re.sub(r'[^\x00-\x7F]+',' ', str(t))
htmltext='\n'.join(['<h1>',t,'</h1>'])
#htmltext='\n'.join([str(x)])
f.write(htmltext)
f.write('\n')
elif(node.attrib['style']=='Section'):
print 'section'
t=findp[i].text
t=re.sub(r'[^\x00-\x7F]+',' ', str(t))
sectioncheck=True
htmltext='\n'.join(['<h2>',str(sectioncount),t,'</h2>'])
sectioncount+=1
#htmltext='\n'.join([str(x)])
f.write(htmltext)
f.write('\n')
elif(node.attrib['style']=='Subsection'):
if(sectioncheck==True):
g=1
g+=1
sectioncheck=False
#print 'subsection',sectioncheck,(sectioncount-1),'.',(g-1)
subsec=findp[i].text
subsec=re.sub(r'[^\x00-\x7F]+',' ', str(subsec))
#scount=(sectioncount-1)+subsectioncount
scount=str((sectioncount-1))+'.'+str((g-1))
subsectioncheck=True
currentlevel=scount
#print str((sectioncount-1)),'.',g
#print 'subsection',scount
htmltext='\n'.join(['<h3>',str(scount),subsec,'</h3>'])
subsectioncount+=0.01
#htmltext='\n'.join([str(x)])
f.write(htmltext)
f.write('\n')
elif(node.attrib['style']=='Subsubsection'):
if(subsectioncheck==True):
g1=1
g1+=1
subsectioncheck=False
#print 'subsection',sectioncheck,(sectioncount-1),'.',(g-1)
subsec=findp[i].text
subsec=re.sub(r'[^\x00-\x7F]+',' ', str(subsec))
#scount=(sectioncount-1)+subsectioncount
scount=str(currentlevel)+'.'+str((g1-1))
print 'subsubsection',scount
#print str((sectioncount-1)),'.',g
#print 'subsection',scount
htmltext='\n'.join(['<h4>',str(scount),subsec,'</h4>'])
subsectioncount+=0.01
#htmltext='\n'.join([str(x)])
f.write(htmltext)
f.write('\n')
else:
htmltext='\n'.join(['<h1>',str(x),'</h1>'])
#htmltext='\n'.join([str(x)])
f.write(htmltext)
f.write('\n')
else:
try:
imagedir=os.path.join(newdir,"Images").replace('\\','/')
if not os.path.exists(imagedir):
os.mkdir(imagedir)
os.chdir(imagedir)
y=imagelist[0]
image=y.text
unique_filename = str(uuid.uuid4())
img = QtGui.QImage()
#image=node.find('Image').text
byteArray = QByteArray.fromBase64(image)
buffer = QBuffer(byteArray)
buffer.open(QIODevice.ReadOnly)
data = QDataStream(buffer)
data >> img
buffer.close()
filename=".".join([unique_filename,"png"])
imgpath="/".join(['static',os.path.basename(newdir),'Images/'])
imgsrc=imgpath+filename
img.save(filename)
imgtag="".join(["<div align=\"center\">","<img src=",imgsrc,">","</div>"])
#print filename
f.write(imgtag)
del imagelist[0]
except:
pass
if (node.tag=='GraphCell' or node.tag=='InputCell'):
## catch the input text
inputtext=node.find('Input').text
'''
if ('simulate' in inputtext):
text='\n'.join(['<p> <b>',inputtext,'</b> </p>'])
f.write(text)
f.write('\n')'''
#print 'arun', inputtext
if(inputtext!=None):
linecount=string.split(inputtext, '\n')
'''
if ('plot(' in inputtext):
text='\n'.join(['<p> <b>',inputtext,'</b> </p> <br>'])
f.write(text)
f.write('\n')
## code to automatically generate plot variable and button in html
plotvar=inputtext.replace('plot','').replace('(','').replace(')','').replace('{','').replace('}','')
listplotvar=plotvar.split(',')
plotid='simulatebutton'+str(count)+'plot'
buttonid='simulatebutton'+str(count)
graphdivid='simulatebutton'+str(count)+'graphdiv'
plotheader="\n".join(['<div>','<select id='+ plotid +' size=5 multiple>', '<option ><b>Select Plot Variables</b> </option>'])
f.write(str(plotheader))
f.write('\n')
for i in xrange(len(listplotvar)):
varname='<option selected>'+ str(listplotvar[i]) + '</option>'
f.write(varname)
f.write('\n')
closeoption="\n".join(['</select> <br>','<button id='+buttonid+'>Simulate</button> <br> <br>' ,'</div>'])
f.write(closeoption)
count=count+1;
else:'''
textid='check'+str(check)+'textarea'
divid='check'+str(check)+'div'
if ('plot(' in inputtext):
text='\n'.join(['<textarea id='+ str(textid)+'>',inputtext,'</textarea> <br> <div id='+divid+'> </div> <br>'])
else:
text='\n'.join(['<textarea id='+ str(textid)+'>',inputtext,'</textarea> <div id='+divid+'> </div> <br>'])
check=check+1
f.write(text)
f.write('\n')
## catch the OMCPLOT datas
curve=node.find('OMCPlot')
if curve!=None:
#count=count+1;
print count
try:
#scriptdata=makeplot(curve,count)
scriptdata=makeplot(curve,divid)
f.write(scriptdata)
f.write('\n')
except:
f.write("No data found")
f.write('\n')
else:
textid='check'+str(check)+'textarea'
divid='check'+str(check)+'div'
inputtext=''
text='\n'.join(['<textarea id='+ str(textid)+'>',inputtext,'</textarea> <div id='+divid+'> </div> <br>'])
check=check+1
f.write(text)
f.write('\n')
print "Empty Graph cells"
f.write('</div></body></html>')
f.close()
print 'Completed'
class qNotebook(QVBoxLayout):
def __init__(self):
QVBoxLayout.__init__(self)
self._teditor = QTextEdit()
self._teditor.setMinimumWidth(500)
self._teditor.setStyleSheet("font: 12pt \"Courier\";")
button_layout = QHBoxLayout()
self.addLayout(button_layout)
self.clear_but = qmy_button(button_layout, self.clear_all, "clear")
self.copy_but = qmy_button(button_layout, self._teditor.copy, "copy")
qmy_button(button_layout, self._teditor.selectAll, "select all")
qmy_button(button_layout, self._teditor.undo, "undo")
qmy_button(button_layout, self._teditor.redo, "redo")
search_button = qButtonWithArgumentsClass("search", self.search_for_text, {"search_text": ""})
button_layout.addWidget(search_button)
qmy_button(button_layout, self.save_as_html, "save notebook")
self.addWidget(self._teditor)
self._teditor.document().setUndoRedoEnabled(True)
self.image_counter = 0
self.image_dict = {}
self.image_data_dict = {}
def append_text(self, text):
self._teditor.append(str(text))
def search_for_text(self, search_text = " "):
self._teditor.find(search_text)
def clear_all(self):
self._teditor.clear()
self.image_dict = {}
self.image_counter = 0
# newdoc = QTextDocument()
# self._teditor.setDocument(newdoc)
def append_image(self, fig=None):
#This assumes that an image is there waiting to be saved from matplotlib
self.imgdata = StringIO.StringIO()
if fig is None:
pyplot.savefig(self.imgdata, transparent = False, format = img_format)
else:
fig.savefig(self.imgdata, transparent = False, format = img_format)
self.abuffer = QBuffer()
self.abuffer.open(QBuffer.ReadWrite)
self.abuffer.write(self.imgdata.getvalue())
self.abuffer.close()
self.abuffer.open(QBuffer.ReadOnly)
iReader = QImageReader(self.abuffer, img_format )
the_image = iReader.read()
# the_image = the_image0.scaledToWidth(figure_width)
# save the image in a file
imageFileName = "image" + str(self.image_counter) + "." + img_format
self.image_data_dict[imageFileName] = self.imgdata
self.image_counter +=1
imageFormat = QTextImageFormat()
imageFormat.setName(imageFileName)
imageFormat.setWidth(image_width)
self.image_dict[imageFileName] = the_image
#insert the image in the text document
text_doc = self._teditor.document()
text_doc.addResource(QTextDocument.ImageResource, QUrl(imageFileName), the_image)
cursor = self._teditor.textCursor()
cursor.movePosition(QTextCursor.End)
cursor.insertImage(imageFormat)
def add_image_data_resource(self, imgdata, imageFileName):
self.abuffer = QBuffer()
self.abuffer.open(QBuffer.ReadWrite)
self.abuffer.write(imgdata.getvalue())
self.abuffer.close()
self.abuffer.open(QBuffer.ReadOnly)
iReader = QImageReader(self.abuffer, img_format )
the_image = iReader.read()
# the_image = the_image0.scaledToWidth(figure_width)
# save the image in a file
# imageFileName = "image" + str(self.image_counter) + "." + img_format
self.image_data_dict[imageFileName] = imgdata
# self.image_counter +=1
imageFormat = QTextImageFormat()
imageFormat.setName(imageFileName)
imageFormat.setWidth(image_width)
self.image_dict[imageFileName] = the_image
#insert the image in the text document
text_doc = self._teditor.document()
text_doc.addResource(QTextDocument.ImageResource, QUrl(imageFileName), the_image)
def append_html_table_from_array(self, a, header_rows=0, precision = 3, caption = None, cmap = None):
nrows = len(a)
ncols = len(a[0])
result_string = "<table border=\"1\" cellspacing=\"0\" cellpadding=\"2\">\n"
if caption != None:
result_string += "<caption>%s</caption>\n" % caption
r = 0
while r < header_rows:
result_string += "<tr>"
for c in range(ncols):
if a[r][c] != "":
# count following blank columns
count = 1
while ((c+count) < len(a[r])) and (a[r][c+count] == "") :
count += 1
val = a[r][c]
if (type(val) == numpy.float64) or (type(val) == float): # @UndefinedVariable
if precision != 999:
val = round(val, precision)
if count > 1:
result_string +="<th colspan=%s>%s</th>" % (count, val)
else:
result_string += "<th>%s</th>" % val
result_string +="</tr>\n"
r += 1
while r < nrows:
result_string += "<tr>"
for c in range(ncols):
val = a[r][c]
if (cmap == None):
fcolor = "#ffffff"
elif (type(val) == int) or (type(val) == float) or (type(val) == numpy.float64): # @UndefinedVariable
fcolor = cmap.color_from_val(val)
else:
fcolor = "#ffffff"
if (val != "") or (c == 0):
if (type(val) == numpy.float64) or (type(val) == float): # @UndefinedVariable
if precision != 999:
val = round(val, precision)
count = 1
while ((c+count) < len(a[r])) and (a[r][c+count] == "") :
count += 1
if count > 1:
result_string +="<td colspan=%s bgcolor=%s>%s</td>" % (count, fcolor, val)
else:
result_string += "<td bgcolor=%s>%s</td>" % (fcolor, val)
result_string +="</tr>\n"
r += 1
result_string += "</table>\n"
self.append_text(result_string)
def create_empty_string_array(self, rows, cols):
table_array = []
for r in range(rows): #@UnusedVariable
the_row = []
for c in range(cols): #@UnusedVariable
the_row.append("")
table_array.append(the_row)
return table_array
def recurse_on_dict_headers(self, sdict, r, c, sorted_headers = None):
if ((type(sdict) != dict) and (type(sdict) != OrderedDict)):
return c + 1
else:
if sorted_headers != None:
sheaders = sorted_headers
else:
sheaders = sorted(sdict.keys())
for k in sheaders:
self.table_array[r][c] = k
c = self.recurse_on_dict_headers(sdict[k], r + 1, c)
return c
def recurse_to_find_size(self, sdict, r, c):
if ((type(sdict) != dict) and (type(sdict) != OrderedDict)):
return r, c + 1
else:
rbiggest = r
for k in sorted(sdict.keys()):
rnew, c = self.recurse_to_find_size(sdict[k], r + 1, c)
if rnew > rbiggest:
rbiggest = rnew
return rbiggest, c
def recurse_on_dict(self, sdict, r, c, sorted_headers = None):
if ((type(sdict) != dict) and (type(sdict) != OrderedDict)):
self.table_array[r][c] = sdict
return c + 1
else:
if sorted_headers != None:
sheaders = sorted_headers
else:
sheaders = sorted(sdict.keys())
for k in sheaders:
c = self.recurse_on_dict(sdict[k], r, c)
return c
def convert_structured_dicts_to_array(self, sdict, sorted_keys = None, sorted_headers = None):
header_levels, ncols = self.recurse_to_find_size(sdict[sdict.keys()[0]], 0, 0)
nrows = header_levels + len(sdict.keys())
self.table_array = self.create_empty_string_array(nrows, ncols)
self.recurse_on_dict_headers(sdict[sdict.keys()[0]], 0, 0, sorted_headers)
if sorted_keys != None:
key_list = sorted_keys
else:
key_list = sdict.keys()
r = header_levels
for entry in key_list:
c = 0
self.table_array[r][0] = entry
self.recurse_on_dict(sdict[entry], r, c, sorted_headers = sorted_headers)
r += 1
return self.table_array
def append_html_table_from_dicts(self, sdict, header_rows = 1, title = None, sorted_keys = None, precision = 3, cmap = None, sorted_headers = None):
the_array = self.convert_structured_dicts_to_array(sdict, sorted_keys, sorted_headers = sorted_headers)
self.append_html_table_from_array(the_array, header_rows, caption = title, precision = precision, cmap = cmap)
def append_table(self, rows, cols, border_style = QTextFrameFormat.BorderStyle_Solid):
tformat = QTextTableFormat()
tformat.setBorderStyle(border_style)
cursor= self._teditor.textCursor()
cursor.movePosition(QTextCursor.End)
table = cursor.insertTable(rows, cols, tformat)
return table
def fill_table_cell(self, row, col, table, text):
cptr = table.cellAt(row, col).firstCursorPosition()
cptr.insertText(text)
def save_as_html(self):
fname = QFileDialog.getSaveFileName()[0]
fdirectoryname = os.path.dirname(fname)
# fdirectoryname = QFileDialog.getExistingDirectory()
# print fdirectoryname
# fname = fdirectoryname + "/report.html"
text_doc = self._teditor.document()
f = open(fname, 'w')
f.write(text_doc.toHtml())
f.close()
for imageFileName in self.image_dict.keys():
full_image_path = fdirectoryname + "/" + imageFileName
iWriter = QImageWriter(full_image_path, img_format)
iWriter.write(self.image_dict[imageFileName])
def main(url, output, option={}):
result = {
"error": [],
"page": {},
"resources": [],
"capture": None,
}
#savedir = appdir + "/artifacts/ghost/" + output
#dump = savedir + "/ghost.pkl"
savedir = os.path.join(appdir, "artifacts/ghost")
dump = savedir + "/" + output
try:
#if os.path.exists(savedir):
# shutil.rmtree(savedir)
#os.makedirs(savedir)
with open(dump, 'wb') as d:
#umsgpack.dump(result, d)
json.dump(result, d)
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
defaults = {
"wait_timeout": 60,
"display": False,
"viewport_size": (800, 600),
"plugins_enabled": True,
"java_enabled": True,
}
proxy_url = None
http_method = "GET"
req_headers = None
body = None
if option:
if "user_agent" in option:
defaults["user_agent"] = str(option["user_agent"])
if "timeout" in option:
defaults["wait_timeout"] = int(option["timeout"])
if "proxy" in option:
proxy_url = option["proxy"]
if "method" in option:
http_method = option["method"]
if "headers" in option:
req_headers = option["headers"]
if "post_data" in option:
body = str(option["post_data"])
logger.info(defaults)
ghost = None
try:
ghost = Ghost(
#log_level=logging.DEBUG,
log_level=logging.INFO,
plugin_path=[
appdir + '/plugins',
'/usr/lib/mozilla/plugins',
],
defaults=defaults,
)
except Exception as e:
logger.error("ghost init failed. " + str(e))
result["error"].append(str(e))
with ghost.start() as session:
if proxy_url:
try:
type = proxy_url.split(":")[0]
server = proxy_url.split("/")[2]
host = server.split(":")[0]
port = server.split(":")[1]
session.set_proxy(
str(type),
host=str(host),
port=int(port),
)
except Exception as e:
logger.debug(e)
headers = {}
if req_headers:
logger.debug(req_headers)
for h in req_headers:
headers[str(h)] = str(req_headers[h])
logger.debug(headers)
if hasattr(ghost, "xvfb"):
logger.info(ghost.xvfb)
page = None
resources = None
try:
page, resources = session.open(url.decode("utf-8"),
method=http_method,
headers=headers,
body=body)
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
#if error:
# result["error"] = error.spilt(".")[-1]
if page:
result["page"] = {
"url": page.url,
"http_status": page.http_status,
"headers": page.headers,
#"content":session.content.encode("utf-8"),
"content": base64.b64encode(session.content.encode("utf-8")),
"seq": 0,
#"error":page.error.encode("utf-8").split(".")[-1],
"error": page.error.split(".")[-1],
}
try:
image = session.capture()
ba = QByteArray()
buffer = QBuffer(ba)
buffer.open(QIODevice.WriteOnly)
image.save(buffer, "PNG")
bio = BytesIO(ba)
bio.seek(0)
#result["capture"] = bio.read()
result["capture"] = base64.b64encode(bio.read())
bio.flush()
bio.close()
ba.clear()
buffer.close()
except Exception as e:
logger.error(str(e))
result["error"].append(str(e))
if resources:
seq = 0
for r in resources:
seq += 1
#logger.debug(r.url)
dict = {
"url": r.url,
"http_status": r.http_status,
"headers": r.headers,
#"content":r.content.encode("utf-8"),
"content": base64.b64encode(r.content),
"seq": seq,
#"error":r.error.encode("utf-8").split(".")[-1],
"error": r.error.split(".")[-1],
}
result["resources"].append(dict)
with open(dump, 'wb') as d:
#umsgpack.dump(result, d)
json.dump(result, d)
logger.debug(dump)
ghost.exit()
return dump
