def handleFinished(self, reply):
headers = []
for header in reply.rawHeaderList():
header = {
'name': str(header),
'value': str(reply.rawHeader(header))
}
headers.append(header)
data = {
'stage': 'end',
'id': self.m_ids[reply],
'url': reply.url().toString(),
'status': reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
'statusText': reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
'contentType': reply.header(QNetworkRequest.ContentTypeHeader),
'redirectURL': reply.header(QNetworkRequest.LocationHeader),
'headers': headers,
'time': QDateTime.currentDateTime()
}
del self.m_ids[reply]
if reply in self.m_started:
del self.m_started[self.m_started.index(reply)]
do_action('NetworkAccessManagerHandleFinished')
self.resourceReceived.emit(data)
def handleFinished(self, reply):
headers = []
for header in reply.rawHeaderList():
header = {"name": str(header), "value": str(reply.rawHeader(header))}
headers.append(header)
data = {
"stage": "end",
"id": self.m_ids[reply],
"url": reply.url().toString(),
"status": reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
"statusText": reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
"contentType": reply.header(QNetworkRequest.ContentTypeHeader),
"redirectURL": reply.header(QNetworkRequest.LocationHeader),
"headers": headers,
"time": QDateTime.currentDateTime(),
}
del self.m_ids[reply]
if reply in self.m_started:
del self.m_started[self.m_started.index(reply)]
do_action("NetworkAccessManagerHandleFinished", Bunch(locals()))
self.resourceReceived.emit(data)
def createRequest(self, op, req, outgoingData):
do_action('NetworkAccessManagerCreateRequestPre')
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors:
reply.ignoreSslErrors()
headers = []
for header in req.rawHeaderList():
header = {
'name': str(header),
'value': str(req.rawHeader(header))
}
headers.append(header)
self.m_idCounter += 1
self.m_ids[reply] = self.m_idCounter
data = {
'id': self.m_idCounter,
'url': req.url().toString(),
'method': self.operationToString(op),
'headers': headers,
'time': QDateTime.currentDateTime()
}
reply.readyRead.connect(self.handleStarted)
do_action('NetworkAccessManagerCreateRequestPost')
self.resourceRequested.emit(data)
return reply
def createRequest(self, op, req, outgoingData):
do_action("NetworkAccessManagerCreateRequestPre", Bunch(locals()))
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors:
reply.ignoreSslErrors()
headers = []
for header in req.rawHeaderList():
header = {"name": str(header), "value": str(req.rawHeader(header))}
headers.append(header)
self.m_idCounter += 1
self.m_ids[reply] = self.m_idCounter
data = {
"id": self.m_idCounter,
"url": req.url().toString(),
"method": toString(op),
"headers": headers,
"time": QDateTime.currentDateTime(),
}
reply.readyRead.connect(self.handleStarted)
do_action("NetworkAccessManagerCreateRequestPost", Bunch(locals()))
self.resourceRequested.emit(data)
return reply
def argParser():
parser = argparse.ArgumentParser(
description="Minimalistic headless WebKit-based JavaScript-driven tool",
usage="%(prog)s [options] script.[js|coffee] [script argument [script argument ...]]",
formatter_class=argparse.RawTextHelpFormatter,
)
parser.add_argument(
"script", metavar="script.[js|coffee]", nargs="?", help="The script to execute, and any args to pass to it"
)
parser.add_argument(
"--disk-cache", default="no", choices=["yes", "no"], help="Enable disk cache (default: %(default)s)"
)
parser.add_argument(
"--ignore-ssl-errors", default="no", choices=["yes", "no"], help="Ignore SSL errors (default: %(default)s)"
)
parser.add_argument(
"--load-images", default="yes", choices=["yes", "no"], help="Load all inlined images (default: %(default)s)"
)
parser.add_argument(
"--load-plugins",
default="no",
choices=["yes", "no"],
help="Load all plugins (i.e. Flash, Silverlight, ...) (default: %(default)s)",
)
parser.add_argument("--proxy", metavar="address:port", help="Set the network proxy")
parser.add_argument("--upload-file", nargs="*", metavar="tag=file", help="Upload 1 or more files")
parser.add_argument("-v", "--verbose", action="store_true", help="Show verbose debug messages")
parser.add_argument("--version", action="version", version=license, help="show this program's version and license")
do_action("ArgParser", Bunch(locals()))
return parser
def parseArgs(args):
# Handle all command-line options
p = argParser()
arg_data = p.parse_known_args(args)
args = arg_data[0]
args.script_args = arg_data[1]
args.disk_cache = False if args.disk_cache == 'no' else True
args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
args.load_images = True if args.load_images == 'yes' else False
args.load_plugins = False if args.load_plugins == 'no' else True
args.local_access_remote = False if args.local_access_remote == 'no' else True
if args.proxy:
item = args.proxy.split(':')
if len(item) < 2 or not len(item[1]):
p.print_help()
sys.exit(1)
args.proxy = item
do_action('ParseArgs')
if not args.script:
p.print_help()
sys.exit(1)
if not os.path.exists(args.script):
sys.exit('No such file or directory: \'%s\'' % args.script)
return args
def handleStarted(self):
reply = self.sender()
if not reply:
return
if reply in self.m_started:
return
self.m_started.append(reply)
headers = []
for header in reply.rawHeaderList():
header = {
'name': str(header),
'value': str(reply.rawHeader(header))
}
headers.append(header)
data = {
'stage': 'start',
'id': self.m_ids[reply],
'url': str(reply.url()),
'status': reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
'statusText': reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
'contentType': reply.header(QNetworkRequest.ContentTypeHeader),
'bodySize': reply.size(),
'redirectURL': reply.header(QNetworkRequest.LocationHeader),
'headers': headers,
'time': QDateTime.currentDateTime()
}
do_action('NetworkAccessManagerHandleStarted')
self.resourceReceived.emit(data)
def createRequest(self, op, req, outgoingData):
if op == QNetworkAccessManager.GetOperation:
qDebug('HTTP/1.1 GET Request')
elif op == QNetworkAccessManager.PostOperation:
qDebug('HTTP/1.1 POST Request')
elif op == QNetworkAccessManager.HeadOperation:
qDebug('HTTP/1.1 HEAD Request')
elif op == QNetworkAccessManager.PutOperation:
qDebug('HTTP/1.1 PUT Request')
elif op == QNetworkAccessManager.DeleteOperation:
qDebug('HTTP/1.1 DELETE Request')
elif op == QNetworkAccessManager.CustomOperation:
qDebug('HTTP/1.1 CUSTOM Request')
else:
qWarning('Unexpected HTTP Operation Type')
qDebug('URL %s' % req.url().toString())
do_action('NetworkAccessManagerCreateRequestPre', Bunch(locals()))
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors == 'yes':
reply.ignoreSslErrors()
do_action('NetworkAccessManagerCreateRequestPost', Bunch(locals()))
return reply
def argParser():
parser = argparse.ArgumentParser(
description='Minimalistic headless WebKit-based JavaScript-driven tool',
usage='%(prog)s [options] script.[js|coffee] [script argument [script argument ...]]',
formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument('script', metavar='script.[js|coffee]', nargs='?',
help='The script to execute, and any args to pass to it'
)
parser.add_argument('--auth', metavar='user:pass',
help='Sets the authentication username and password'
)
parser.add_argument('--config', metavar='/path/to/config',
help='Specifies path to a JSON-formatted config file'
)
parser.add_argument('--cookies', metavar='/path/to/cookieJar',
help='Use persistent cookies from an INI-formatted CookieJar file'
)
parser.add_argument('--disk-cache', default='no',
choices=['yes', 'no'],
help='Enable disk cache (default: %(default)s)'
)
parser.add_argument('--ignore-ssl-errors', default='no',
choices=['yes', 'no'],
help='Ignore SSL errors (default: %(default)s)'
)
parser.add_argument('--load-images', default='yes',
choices=['yes', 'no'],
help='Load all inlined images (default: %(default)s)'
)
parser.add_argument('--load-plugins', default='no',
choices=['yes', 'no'],
help='Load all plugins (i.e. Flash, Silverlight, ...) (default: %(default)s)'
)
parser.add_argument('--local-access-remote', default='no',
choices=['yes', 'no'],
help='Local content can access remote URL (default: %(default)s)'
)
parser.add_argument('--output-encoding', default='System', metavar='encoding',
help='Sets the encoding used for terminal output (default: %(default)s)'
)
parser.add_argument('--proxy', metavar='address:port',
help='Set the network proxy'
)
parser.add_argument('--script-encoding', default='utf-8', metavar='encoding',
help='Sets the encoding used for scripts (default: %(default)s)'
)
parser.add_argument('-v', '--verbose', action='store_true',
help='Show verbose debug messages'
)
parser.add_argument('--version',
action='version', version=license,
help="show this program's version and license"
)
do_action('ArgParser')
return parser
def createRequest(self, op, req, outgoingData):
do_action('NetworkAccessManagerCreateRequestPre')
reply = QNetworkAccessManager.createRequest(self, op, req,
outgoingData)
if self.m_ignoreSslErrors:
reply.ignoreSslErrors()
headers = []
for header in req.rawHeaderList():
header = {'name': str(header), 'value': str(req.rawHeader(header))}
headers.append(header)
self.m_idCounter += 1
self.m_ids[reply] = self.m_idCounter
data = {
'id': self.m_idCounter,
'url': req.url().toString(),
'method': toString(op),
'headers': headers,
'time': QDateTime.currentDateTime()
}
reply.readyRead.connect(self.handleStarted)
do_action('NetworkAccessManagerCreateRequestPost')
self.resourceRequested.emit(data)
return reply
def __init__(self, parent, args):
super(NetworkAccessManager, self).__init__(parent)
self.m_userName = self.m_password = ''
self.m_ignoreSslErrors = args.ignore_ssl_errors
self.m_idCounter = 0
self.m_ids = {}
self.m_started = []
if args.cookies_file:
self.setCookieJar(CookieJar(self, args.cookies_file))
if args.disk_cache:
m_networkDiskCache = QNetworkDiskCache()
m_networkDiskCache.setCacheDirectory(
QDesktopServices.storageLocation(
QDesktopServices.CacheLocation))
if args.max_disk_cache_size > 0:
m_networkDiskCache.setMaximumCacheSize(
args.max_disk_cache_size * 1024)
self.setCache(m_networkDiskCache)
self.authenticationRequired.connect(self.provideAuthentication)
self.finished.connect(self.handleFinished)
do_action('NetworkAccessManagerInit')
def parseArgs(args):
# Handle all command-line options
p = argParser()
arg_data = p.parse_known_args(args)
args = arg_data[0]
args.script_args = arg_data[1]
args.disk_cache = False if args.disk_cache == 'no' else True
args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
args.load_images = True if args.load_images == 'yes' else False
args.load_plugins = False if args.load_plugins == 'no' else True
args.local_access_remote = False if args.local_access_remote == 'no' else True
if args.proxy:
item = args.proxy.split(':')
if len(item) < 2 or not len(item[1]):
p.print_help()
sys.exit(1)
args.proxy = item
if args.cookies and not os.path.exists(args.cookies):
sys.exit("No such file or directory: '%s'" % args.cookies)
do_action('ParseArgs')
if not args.script:
p.print_help()
sys.exit(1)
if not os.path.exists(args.script):
sys.exit("No such file or directory: '%s'" % args.script)
return args
def parseArgs(args):
# Handle all command-line options
p = argParser()
arg_data = p.parse_known_args(args)
args = arg_data[0]
args.script_args = arg_data[1]
args.disk_cache = False if args.disk_cache == 'no' else True
args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
args.load_images = True if args.load_images == 'yes' else False
args.load_plugins = False if args.load_plugins == 'no' else True
if args.proxy:
item = args.proxy.split(':')
if len(item) < 2 or not len(item[1]):
p.print_help()
sys.exit(1)
args.proxy = item
do_action('ParseArgs', Bunch(locals()))
if not args.script:
p.print_help()
sys.exit(1)
try:
with codecs.open(args.script, encoding='utf-8') as script:
args.script_name = script.name
args.script = script.read()
except IOError as (errno, stderr):
sys.exit('%s: \'%s\'' % (stderr, args.script))
def __init__(self, args, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_defaultPageSettings = {}
self.m_pages = []
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_returnValue = 0
self.m_terminated = False
# setup the values from args
self.m_scriptFile = args.script
self.m_args = args.script_args
self.m_filesystem = FileSystem(self)
self.m_pages.append(self.m_page)
do_action('PhantomInitPre')
if not args.proxy:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0],
int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
# Provide WebPage with a non-standard Network Access Manager
self.m_netAccessMan = NetworkAccessManager(args.disk_cache,
args.ignore_ssl_errors,
self)
self.m_page.setNetworkAccessManager(self.m_netAccessMan)
self.m_page.javaScriptConsoleMessageSent.connect(
self.printConsoleMessage)
self.m_defaultPageSettings['loadImages'] = args.load_images
self.m_defaultPageSettings['loadPlugins'] = args.load_plugins
self.m_defaultPageSettings['userAgent'] = self.m_page.userAgent()
self.m_defaultPageSettings[
'localAccessRemote'] = args.local_access_remote
self.m_page.applySettings(self.m_defaultPageSettings)
self.libraryPath = os.path.dirname(os.path.abspath(self.m_scriptFile))
# inject our properties and slots into javascript
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
self.m_page.mainFrame().addToJavaScriptWindowObject(
'fs', self.m_filesystem)
bootstrap = QFile(':/bootstrap.js')
if not bootstrap.open(QFile.ReadOnly):
sys.exit('Can not bootstrap!')
bootstrapper = str(bootstrap.readAll())
bootstrap.close()
if not bootstrapper:
sys.exit('Can not bootstrap!')
self.m_page.mainFrame().evaluateJavaScript(bootstrapper)
do_action('PhantomInitPost')
def __init__(self, parent):
super(CustomPage, self).__init__(parent)
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
self.m_uploadFile = ''
do_action('CustomPageInit')
def __init__(self, conn):
super(WebServerResponse, self).__init__()
self.setObjectName("WebServerResponse")
self.m_conn = conn
do_action("WebServerResponseInit")
def __init__(self, conn):
super(WebServerResponse, self).__init__()
self.setObjectName('WebServerResponse')
self.m_conn = conn
do_action('WebServerResponseInit')
def parseArgs(app, args):
# Handle all command-line options
p = argParser()
arg_data = p.parse_known_args(args)
args = arg_data[0]
args.script_args = arg_data[1]
# convert script args to unicode
for i, arg in enumerate(args.script_args):
args.script_args[i] = unicode(arg, 'utf-8')
# register an alternative Message Handler
messageHandler = MessageHandler(args.verbose)
qInstallMsgHandler(messageHandler.process)
file_check = (args.cookies_file, args.config)
for file_ in file_check:
if file_ is not None and not os.path.exists(file_):
sys.exit("No such file or directory: '%s'" % file_)
if args.config:
config = Config(app, args.config)
# apply settings
for setting in config.settings:
setattr(args, config.settings[setting]['mapping'], config.property(setting))
split_check = (
(args.proxy, 'proxy'),
)
for arg, name in split_check:
if arg:
item = arg.split(':')
if len(item) < 2 or not len(item[1]):
p.print_help()
sys.exit(1)
setattr(args, name, item)
if args.proxy is not None:
if args.proxy_type == 'socks5':
args.proxy_type = QNetworkProxy.Socks5Proxy
do_action('ParseArgs', args)
if args.debug:
debug(args.debug)
# verbose flag got changed on us, so we reload the flag
if messageHandler.verbose != args.verbose:
messageHandler.verbose = args.verbose
if args.script is None:
p.print_help()
sys.exit(1)
if not os.path.exists(args.script):
sys.exit("No such file or directory: '%s'" % args.script)
return args
def __init__(self, parent=None):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
self.m_uploadFile = ''
do_action('CustomPageInit', Bunch(locals()))
def __init__(self, parent=None):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
self.m_uploadFile = ''
do_action('CustomPageInit')
def parseArgs(app, args):
# Handle all command-line options
p = argParser()
arg_data = p.parse_known_args(args)
args = arg_data[0]
args.script_args = arg_data[1]
# convert script args to unicode
for i, arg in enumerate(args.script_args):
args.script_args[i] = unicode(arg, 'utf-8')
# register an alternative Message Handler
messageHandler = MessageHandler(args.verbose)
qInstallMsgHandler(messageHandler.process)
file_check = (args.cookies_file, args.config)
for file_ in file_check:
if file_ is not None and not os.path.exists(file_):
sys.exit("No such file or directory: '%s'" % file_)
if args.config:
config = Config(app, args.config)
# apply settings
for setting in config.settings:
setattr(args, config.settings[setting]['mapping'],
config.property(setting))
split_check = ((args.proxy, 'proxy'), )
for arg, name in split_check:
if arg:
item = arg.split(':')
if len(item) < 2 or not len(item[1]):
p.print_help()
sys.exit(1)
setattr(args, name, item)
if args.proxy is not None:
if args.proxy_type == 'socks5':
args.proxy_type = QNetworkProxy.Socks5Proxy
do_action('ParseArgs', args)
if args.debug:
debug(args.debug)
# verbose flag got changed on us, so we reload the flag
if messageHandler.verbose != args.verbose:
messageHandler.verbose = args.verbose
if args.script is None:
p.print_help()
sys.exit(1)
if not os.path.exists(args.script):
sys.exit("No such file or directory: '%s'" % args.script)
return args
def __init__(self, parent, args):
QObject.__init__(self, parent)
# variable declarations
self.m_defaultPageSettings = {}
self.m_pages = []
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_returnValue = 0
self.m_terminated = False
# setup the values from args
self.m_scriptFile = args.script
self.m_args = args.script_args
self.m_filesystem = FileSystem(self)
self.m_pages.append(self.m_page)
do_action('PhantomInitPre')
if args.proxy is None:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0], int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
# Provide WebPage with a non-standard Network Access Manager
self.m_netAccessMan = NetworkAccessManager(self, args.disk_cache, args.cookies, args.ignore_ssl_errors)
self.m_page.setNetworkAccessManager(self.m_netAccessMan)
self.m_page.javaScriptConsoleMessageSent.connect(self.printConsoleMessage)
self.m_defaultPageSettings['loadImages'] = args.load_images
self.m_defaultPageSettings['loadPlugins'] = args.load_plugins
self.m_defaultPageSettings['javascriptEnabled'] = True
self.m_defaultPageSettings['XSSAuditingEnabled'] = False
self.m_defaultPageSettings['userAgent'] = self.m_page.userAgent()
self.m_defaultPageSettings['localAccessRemote'] = args.local_access_remote
self.m_page.applySettings(self.m_defaultPageSettings)
self.libraryPath = os.path.dirname(os.path.abspath(self.m_scriptFile))
# inject our properties and slots into javascript
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
self.m_page.mainFrame().addToJavaScriptWindowObject('fs', self.m_filesystem)
bootstrap = QFile(':/bootstrap.js')
if not bootstrap.open(QFile.ReadOnly):
sys.exit('Can not bootstrap!')
bootstrapper = str(bootstrap.readAll())
bootstrap.close()
if not bootstrapper:
sys.exit('Can not bootstrap!')
self.m_page.mainFrame().evaluateJavaScript(bootstrapper)
do_action('PhantomInitPost')
def __init__(self, parent):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
self.m_scrollPosition = QPoint()
self.m_uploadFile = ''
do_action('CustomPageInit')
def __init__(self, request):
super(WebServerRequest, self).__init__()
self.setObjectName('WebServerRequest')
self.m_request = request
self.m_headerNames = list(request.headers)
self.m_headers = CaseInsensitiveDict(request.headers)
self.m_numHeaders = len(request.headers)
do_action('WebServerRequestInit')
def __init__(self, parent):
super(WebServer, self).__init__(parent)
self.setObjectName('WebServer')
self.m_port = 0
self.httpd = None
servers.append(self)
do_action('WebServerInit')
def __init__(self, args, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_loadStatus = self.m_state = ''
self.m_var = self.m_paperSize = self.m_loadScript_cache = {}
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_clipRect = QRect()
# setup the values from args
self.m_script = args.script.read()
self.m_scriptFile = args.script.name
self.m_scriptDir = os.path.dirname(args.script.name) + '/'
self.m_args = args.script_args
self.m_upload_file = args.upload_file
autoLoadImages = False if args.load_images == 'no' else True
pluginsEnabled = True if args.load_plugins == 'yes' else False
args.script.close()
do_action('PhantomInitPre', Bunch(locals()))
palette = self.m_page.palette()
palette.setBrush(QPalette.Base, Qt.transparent)
self.m_page.setPalette(palette)
if not args.proxy:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0], int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
self.m_page.settings().setAttribute(QWebSettings.AutoLoadImages, autoLoadImages)
self.m_page.settings().setAttribute(QWebSettings.PluginsEnabled, pluginsEnabled)
self.m_page.settings().setAttribute(QWebSettings.FrameFlatteningEnabled, True)
self.m_page.settings().setAttribute(QWebSettings.OfflineStorageDatabaseEnabled, True)
self.m_page.settings().setAttribute(QWebSettings.LocalStorageEnabled, True)
self.m_page.settings().setLocalStoragePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_page.settings().setOfflineStoragePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
# Ensure we have a document.body.
self.m_page.mainFrame().setHtml('<html><body></body></html>')
self.m_page.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.m_page.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
m_netAccessMan = NetworkAccessManager(args.disk_cache, args.ignore_ssl_errors, self)
self.m_page.setNetworkAccessManager(m_netAccessMan)
# inject our properties and slots into javascript
self.m_page.mainFrame().javaScriptWindowObjectCleared.connect(self.inject)
self.m_page.loadFinished.connect(self.finish)
do_action('PhantomInitPost', Bunch(locals()))
def __init__(self, parent):
super(WebServer, self).__init__(parent)
self.setObjectName("WebServer")
self.m_port = 0
self.httpd = None
servers.append(self)
do_action("WebServerInit")
def parseArgs(app, args):
# Handle all command-line options
p = argParser()
arg_data = p.parse_known_args(args)
args = arg_data[0]
args.script_args = arg_data[1]
args.disk_cache = False if args.disk_cache == 'no' else True
args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True
args.load_images = True if args.load_images == 'yes' else False
args.load_plugins = False if args.load_plugins == 'no' else True
args.local_access_remote = False if args.local_access_remote == 'no' else True
# register an alternative Message Handler
messageHandler = MessageHandler(args.verbose)
qInstallMsgHandler(messageHandler.process)
file_check = (args.cookies, args.config)
for file_ in file_check:
if file_ is not None and not os.path.exists(file_):
sys.exit("No such file or directory: '%s'" % file_)
if args.config:
config = Config(app, args.config)
# apply settings
for setting in config.settings:
setattr(args, config.settings[setting]['mapping'], config.property(setting))
# special case for verbose arg, which will need to be re-applied
if setting == 'verbose':
messageHandler.verbose = args.verbose
split_check = (
(args.proxy, 'proxy'),
(args.auth, 'auth')
)
for arg, name in split_check:
if arg:
item = arg.split(':')
if len(item) < 2 or not len(item[1]):
p.print_help()
sys.exit(1)
setattr(args, name, item)
do_action('ParseArgs')
if args.script is None:
p.print_help()
sys.exit(1)
if not os.path.exists(args.script):
sys.exit("No such file or directory: '%s'" % args.script)
return args
def __init__(self, request):
super(WebServerRequest, self).__init__()
self.setObjectName("WebServerRequest")
self.m_request = request
self.m_headerNames = list(request.headers)
self.m_headers = CaseInsensitiveDict(request.headers)
self.m_numHeaders = len(request.headers)
do_action("WebServerRequestInit")
def __init__(self, parent=None):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_nextFileTag = ''
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
if self.parent.m_verbose:
self.currentFrame().urlChanged.connect(self.handleFrameUrlChanged)
self.linkClicked.connect(self.handleLinkClicked)
do_action('WebPageInit', Bunch(locals()))
def handleFinished(self, reply):
qDebug('HTTP/1.1 Response')
qDebug('URL %s' % reply.url().toString())
code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if code:
qDebug('Status code: %d' % code)
do_action('NetworkAccessManagerHandleFinished', Bunch(locals()))
headerPairs = reply.rawHeaderPairs()
for pair in headerPairs:
qDebug('"%s" = "%s"' % (pair[0], pair[1]))
def __init__(self, parent=None):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_nextFileTag = ""
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
if self.parent.m_verbose:
self.currentFrame().urlChanged.connect(self.handleFrameUrlChanged)
self.linkClicked.connect(self.handleLinkClicked)
do_action("WebPageInit", Bunch(locals()))
def argParser():
parser = argparse.ArgumentParser(
description='Minimalistic headless WebKit-based JavaScript-driven tool',
usage=
'%(prog)s [options] script.[js|coffee] [script argument [script argument ...]]',
formatter_class=argparse.RawTextHelpFormatter)
parser.add_argument(
'script',
metavar='script.[js|coffee]',
nargs='?',
help='The script to execute, and any args to pass to it')
parser.add_argument('--disk-cache',
default='no',
choices=['yes', 'no'],
help='Enable disk cache (default: %(default)s)')
parser.add_argument('--ignore-ssl-errors',
default='no',
choices=['yes', 'no'],
help='Ignore SSL errors (default: %(default)s)')
parser.add_argument('--load-images',
default='yes',
choices=['yes', 'no'],
help='Load all inlined images (default: %(default)s)')
parser.add_argument(
'--load-plugins',
default='no',
choices=['yes', 'no'],
help=
'Load all plugins (i.e. Flash, Silverlight, ...) (default: %(default)s)'
)
parser.add_argument(
'--local-access-remote',
default='no',
choices=['yes', 'no'],
help='Local content can access remote URL (default: %(default)s)')
parser.add_argument('--proxy',
metavar='address:port',
help='Set the network proxy')
parser.add_argument('-v',
'--verbose',
action='store_true',
help='Show verbose debug messages')
parser.add_argument('--version',
action='version',
version=license,
help='show this program\'s version and license')
do_action('ArgParser', Bunch(locals()))
return parser
def __init__(self, diskCacheEnabled, ignoreSslErrors, parent=None):
QNetworkAccessManager.__init__(self, parent)
self.m_ignoreSslErrors = ignoreSslErrors
if parent.m_verbose:
self.finished.connect(self.handleFinished)
if diskCacheEnabled == 'yes':
m_networkDiskCache = QNetworkDiskCache()
m_networkDiskCache.setCacheDirectory(QDesktopServices.storageLocation(QDesktopServices.CacheLocation))
self.setCache(m_networkDiskCache)
do_action('NetworkAccessManagerInit', Bunch(locals()))
def __init__(self, parent, args):
super(Phantom, self).__init__(parent)
# variable declarations
self.m_defaultPageSettings = {}
self.m_pages = []
self.m_servers = []
self.m_verbose = args.verbose
self.m_page = WebPage(self, args)
self.m_returnValue = 0
self.m_terminated = False
# setup the values from args
self.app_args = args
self.m_scriptFile = args.script
self.m_args = args.script_args
self.m_scriptEncoding = Encode(args.script_encoding, 'utf-8')
self.m_outputEncoding = Encode(args.output_encoding,
sys.stdout.encoding_sys)
self.m_pages.append(self.m_page)
do_action('PhantomInitPre')
if args.proxy is None:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(args.proxy_type, args.proxy[0],
int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
self.m_page.javaScriptConsoleMessageSent.connect(
self.printConsoleMessage)
self.m_defaultPageSettings['loadImages'] = args.load_images
self.m_defaultPageSettings['loadPlugins'] = args.load_plugins
self.m_defaultPageSettings['javascriptEnabled'] = True
self.m_defaultPageSettings['XSSAuditingEnabled'] = False
self.m_defaultPageSettings['userAgent'] = self.m_page.userAgent()
self.m_defaultPageSettings[
'localToRemoteUrlAccessEnabled'] = args.local_to_remote_url_access
self.m_page.applySettings(self.m_defaultPageSettings)
self.libraryPath = os.path.dirname(os.path.abspath(self.m_scriptFile))
# inject our properties and slots into javascript
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
with QPyFile(':/bootstrap.js') as f:
self.m_page.mainFrame().evaluateJavaScript(f.readAll())
do_action('PhantomInitPost')
def __init__(self, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_paperSize = {}
self.m_clipRect = QRect()
self.m_libraryPath = ''
self.setObjectName('WebPage')
self.m_webPage = CustomPage(self)
self.m_mainFrame = self.m_webPage.mainFrame()
self.m_webPage.loadStarted.connect(self.loadStarted)
self.m_webPage.loadFinished.connect(self.finish)
# Start with transparent background
palette = self.m_webPage.palette()
palette.setBrush(QPalette.Base, Qt.transparent)
self.m_webPage.setPalette(palette)
# Page size does not need to take scrollbars into account
self.m_webPage.mainFrame().setScrollBarPolicy(Qt.Horizontal,
Qt.ScrollBarAlwaysOff)
self.m_webPage.mainFrame().setScrollBarPolicy(Qt.Vertical,
Qt.ScrollBarAlwaysOff)
self.m_webPage.settings().setAttribute(
QWebSettings.OfflineStorageDatabaseEnabled, True)
self.m_webPage.settings().setOfflineStoragePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_webPage.settings().setAttribute(
QWebSettings.LocalStorageDatabaseEnabled, True)
self.m_webPage.settings().setAttribute(
QWebSettings.OfflineWebApplicationCacheEnabled, True)
self.m_webPage.settings().setOfflineWebApplicationCachePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_webPage.settings().setAttribute(
QWebSettings.FrameFlatteningEnabled, True)
self.m_webPage.settings().setAttribute(
QWebSettings.LocalStorageEnabled, True)
self.m_webPage.settings().setLocalStoragePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
# Ensure we have a document.body.
self.m_webPage.mainFrame().setHtml('<html><body></body></html>')
self.m_webPage.setViewportSize(QSize(400, 300))
do_action('WebPageInit', Bunch(locals()))
def __init__(self, args, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_defaultPageSettings = {}
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_returnValue = 0
self.m_terminated = False
# setup the values from args
self.m_script = args.script
self.m_scriptFile = args.script_name
self.m_args = args.script_args
do_action('PhantomInitPre', Bunch(locals()))
if not args.proxy:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0], int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
# Provide WebPage with a non-standard Network Access Manager
self.m_netAccessMan = NetworkAccessManager(args.disk_cache, args.ignore_ssl_errors, self)
self.m_page.setNetworkAccessManager(self.m_netAccessMan)
self.m_page.javaScriptConsoleMessageSent.connect(self.printConsoleMessage)
self.m_defaultPageSettings['loadImages'] = args.load_images
self.m_defaultPageSettings['loadPlugins'] = args.load_plugins
self.m_defaultPageSettings['userAgent'] = self.m_page.userAgent()
self.m_page.applySettings(self.m_defaultPageSettings)
# inject our properties and slots into javascript
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
bootstrap = QFile(':/bootstrap.js')
if not bootstrap.open(QFile.ReadOnly):
qCritical('Can not bootstrap!')
sys.exit(1)
bootstrapper = str(bootstrap.readAll())
bootstrap.close()
if not bootstrapper:
qCritical('Can not bootstrap!')
sys.exit(1)
self.m_page.mainFrame().evaluateJavaScript(bootstrapper)
do_action('PhantomInitPost', Bunch(locals()))
def argParser():
parser = argparse.ArgumentParser(
description='Minimalistic headless WebKit-based JavaScript-driven tool',
usage='%(prog)s [options] script.[js|coffee] [script argument [script argument ...]]',
formatter_class=argparse.RawTextHelpFormatter
)
parser.add_argument('script', metavar='script.[js|coffee]', nargs='?',
help='The script to execute, and any args to pass to it'
)
parser.add_argument('--disk-cache', default='no',
choices=['yes', 'no'],
help='Enable disk cache (default: %(default)s)'
)
parser.add_argument('--cookies', metavar='CookieJar',
help='Use persistent cookies from a CookieJar INI file'
)
parser.add_argument('--ignore-ssl-errors', default='no',
choices=['yes', 'no'],
help='Ignore SSL errors (default: %(default)s)'
)
parser.add_argument('--load-images', default='yes',
choices=['yes', 'no'],
help='Load all inlined images (default: %(default)s)'
)
parser.add_argument('--load-plugins', default='no',
choices=['yes', 'no'],
help='Load all plugins (i.e. Flash, Silverlight, ...) (default: %(default)s)'
)
parser.add_argument('--local-access-remote', default='no',
choices=['yes', 'no'],
help='Local content can access remote URL (default: %(default)s)'
)
parser.add_argument('--proxy', metavar='address:port',
help='Set the network proxy'
)
parser.add_argument('-v', '--verbose', action='store_true',
help='Show verbose debug messages'
)
parser.add_argument('--version',
action='version', version=license,
help='show this program\'s version and license'
)
do_action('ArgParser')
return parser
def __init__(self, diskCacheEnabled, ignoreSslErrors, parent=None):
QNetworkAccessManager.__init__(self, parent)
self.m_ignoreSslErrors = ignoreSslErrors
self.m_idCounter = 0
self.m_ids = {}
self.m_started = []
self.finished.connect(self.handleFinished)
if diskCacheEnabled:
m_networkDiskCache = QNetworkDiskCache()
m_networkDiskCache.setCacheDirectory(QDesktopServices.storageLocation(QDesktopServices.CacheLocation))
self.setCache(m_networkDiskCache)
do_action('NetworkAccessManagerInit')
class CustomPage(QWebPage):
def __init__(self, parent=None):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
self.m_uploadFile = ''
do_action('CustomPageInit', Bunch(locals()))
def chooseFile(self, originatingFrame, oldFile):
return self.m_uploadFile
def shouldInterruptJavaScript(self):
QApplication.processEvents(QEventLoop.AllEvents, 42)
return False
def javaScriptAlert(self, originatingFrame, msg):
self.parent.javaScriptAlertSent.emit(msg)
def javaScriptConsoleMessage(self, message, lineNumber, sourceID):
self.parent.javaScriptConsoleMessageSent.emit(message, lineNumber,
sourceID)
def userAgentForUrl(self, url):
return self.m_userAgent
do_action('CustomPage', Bunch(locals()))
def __init__(self, parent, args):
super(Phantom, self).__init__(parent)
# variable declarations
self.m_defaultPageSettings = {}
self.m_pages = []
self.m_verbose = args.verbose
self.m_page = WebPage(self, args)
self.m_returnValue = 0
self.m_terminated = False
# setup the values from args
self.app_args = args
self.m_scriptFile = args.script
self.m_args = args.script_args
self.m_scriptEncoding = Encode(args.script_encoding, 'utf-8')
self.m_outputEncoding = Encode(args.output_encoding, sys.stdout.encoding_sys)
self.m_pages.append(self.m_page)
do_action('PhantomInitPre')
if args.proxy is None:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0], int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
self.m_page.javaScriptConsoleMessageSent.connect(self.printConsoleMessage)
self.m_defaultPageSettings['loadImages'] = args.load_images
self.m_defaultPageSettings['loadPlugins'] = args.load_plugins
self.m_defaultPageSettings['javascriptEnabled'] = True
self.m_defaultPageSettings['XSSAuditingEnabled'] = False
self.m_defaultPageSettings['userAgent'] = self.m_page.userAgent()
self.m_defaultPageSettings['localToRemoteUrlAccessEnabled'] = args.local_to_remote_url_access
self.m_page.applySettings(self.m_defaultPageSettings)
self.libraryPath = os.path.dirname(os.path.abspath(self.m_scriptFile))
# inject our properties and slots into javascript
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
with QPyFile(':/bootstrap.js') as f:
bootstrap = str(f.readAll())
self.m_page.mainFrame().evaluateJavaScript(bootstrap)
do_action('PhantomInitPost')
def __init__(self, parent, args):
super(WebPage, self).__init__(parent)
# variable declarations
self.m_paperSize = {}
self.m_clipRect = QRect()
self.m_libraryPath = ''
self.m_scrollPosition = QPoint()
self.setObjectName('WebPage')
self.m_webPage = CustomPage(self)
self.m_mainFrame = self.m_webPage.mainFrame()
self.m_mainFrame.javaScriptWindowObjectCleared.connect(self.initialized)
self.m_webPage.loadStarted.connect(self.loadStarted)
self.m_webPage.loadFinished.connect(self.finish)
# Start with transparent background
palette = self.m_webPage.palette()
palette.setBrush(QPalette.Base, Qt.transparent)
self.m_webPage.setPalette(palette)
# Page size does not need to take scrollbars into account
self.m_webPage.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.m_webPage.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.m_webPage.settings().setAttribute(QWebSettings.OfflineStorageDatabaseEnabled, True)
self.m_webPage.settings().setOfflineStoragePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_webPage.settings().setAttribute(QWebSettings.LocalStorageDatabaseEnabled, True)
self.m_webPage.settings().setAttribute(QWebSettings.OfflineWebApplicationCacheEnabled, True)
self.m_webPage.settings().setOfflineWebApplicationCachePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_webPage.settings().setAttribute(QWebSettings.FrameFlatteningEnabled, True)
self.m_webPage.settings().setAttribute(QWebSettings.LocalStorageEnabled, True)
self.m_webPage.settings().setLocalStoragePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
# Ensure we have a document.body.
self.m_webPage.mainFrame().setHtml(self.blankHtml)
# Custom network access manager to allow traffic monitoring
self.m_networkAccessManager = NetworkAccessManager(self.parent(), args)
self.m_webPage.setNetworkAccessManager(self.m_networkAccessManager)
self.m_networkAccessManager.resourceRequested.connect(self.resourceRequested)
self.m_networkAccessManager.resourceReceived.connect(self.resourceReceived)
self.m_webPage.setViewportSize(QSize(400, 300))
do_action('WebPageInit')
class NetworkAccessManager(QNetworkAccessManager):
def __init__(self, diskCacheEnabled, ignoreSslErrors, parent=None):
QNetworkAccessManager.__init__(self, parent)
self.m_ignoreSslErrors = ignoreSslErrors
if parent.m_verbose:
self.finished.connect(self.handleFinished)
if diskCacheEnabled == 'yes':
m_networkDiskCache = QNetworkDiskCache()
m_networkDiskCache.setCacheDirectory(QDesktopServices.storageLocation(QDesktopServices.CacheLocation))
self.setCache(m_networkDiskCache)
do_action('NetworkAccessManagerInit', Bunch(locals()))
def createRequest(self, op, req, outgoingData):
if op == QNetworkAccessManager.GetOperation:
qDebug('HTTP/1.1 GET Request')
elif op == QNetworkAccessManager.PostOperation:
qDebug('HTTP/1.1 POST Request')
elif op == QNetworkAccessManager.HeadOperation:
qDebug('HTTP/1.1 HEAD Request')
elif op == QNetworkAccessManager.PutOperation:
qDebug('HTTP/1.1 PUT Request')
elif op == QNetworkAccessManager.DeleteOperation:
qDebug('HTTP/1.1 DELETE Request')
elif op == QNetworkAccessManager.CustomOperation:
qDebug('HTTP/1.1 CUSTOM Request')
else:
qWarning('Unexpected HTTP Operation Type')
qDebug('URL %s' % req.url().toString())
do_action('NetworkAccessManagerCreateRequestPre', Bunch(locals()))
reply = QNetworkAccessManager.createRequest(self, op, req, outgoingData)
if self.m_ignoreSslErrors == 'yes':
reply.ignoreSslErrors()
do_action('NetworkAccessManagerCreateRequestPost', Bunch(locals()))
return reply
def handleFinished(self, reply):
qDebug('HTTP/1.1 Response')
qDebug('URL %s' % reply.url().toString())
code = reply.attribute(QNetworkRequest.HttpStatusCodeAttribute)
if code:
qDebug('Status code: %d' % code)
do_action('NetworkAccessManagerHandleFinished', Bunch(locals()))
headerPairs = reply.rawHeaderPairs()
for pair in headerPairs:
qDebug('"%s" = "%s"' % (pair[0], pair[1]))
do_action('NetworkAccessManager', Bunch(locals()))
def __init__(self, parent):
QObject.__init__(self, parent)
self.m_parent = parent
# variable declarations
self.m_paperSize = {}
self.m_clipRect = QRect()
self.m_libraryPath = ""
self.m_mousePos = QPoint()
self.setObjectName("WebPage")
self.m_webPage = CustomPage(self)
self.m_mainFrame = self.m_webPage.mainFrame()
self.m_mainFrame.javaScriptWindowObjectCleared.connect(self.initialized)
self.m_webPage.loadStarted.connect(self.loadStarted)
self.m_webPage.loadFinished.connect(self.finish)
# Start with transparent background
palette = self.m_webPage.palette()
palette.setBrush(QPalette.Base, Qt.transparent)
self.m_webPage.setPalette(palette)
# Page size does not need to take scrollbars into account
self.m_webPage.mainFrame().setScrollBarPolicy(Qt.Horizontal, Qt.ScrollBarAlwaysOff)
self.m_webPage.mainFrame().setScrollBarPolicy(Qt.Vertical, Qt.ScrollBarAlwaysOff)
self.m_webPage.settings().setAttribute(QWebSettings.OfflineStorageDatabaseEnabled, True)
self.m_webPage.settings().setOfflineStoragePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_webPage.settings().setAttribute(QWebSettings.LocalStorageDatabaseEnabled, True)
self.m_webPage.settings().setAttribute(QWebSettings.OfflineWebApplicationCacheEnabled, True)
self.m_webPage.settings().setOfflineWebApplicationCachePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation)
)
self.m_webPage.settings().setAttribute(QWebSettings.FrameFlatteningEnabled, True)
self.m_webPage.settings().setAttribute(QWebSettings.LocalStorageEnabled, True)
self.m_webPage.settings().setLocalStoragePath(QDesktopServices.storageLocation(QDesktopServices.DataLocation))
# Ensure we have a document.body.
self.m_webPage.mainFrame().setHtml("<html><body></body></html>")
self.m_webPage.setViewportSize(QSize(400, 300))
do_action("WebPageInit")
def __init__(self, diskCacheEnabled, ignoreSslErrors, parent=None):
QNetworkAccessManager.__init__(self, parent)
self.m_ignoreSslErrors = ignoreSslErrors
self.m_idCounter = 0
self.m_ids = {}
self.m_started = []
self.finished.connect(self.handleFinished)
if diskCacheEnabled:
m_networkDiskCache = QNetworkDiskCache()
m_networkDiskCache.setCacheDirectory(
QDesktopServices.storageLocation(
QDesktopServices.CacheLocation))
self.setCache(m_networkDiskCache)
do_action('NetworkAccessManagerInit')
def main():
app = QApplication(sys.argv)
app.setWindowIcon(QIcon(':/resources/pyphantomjs-icon.png'))
app.setApplicationName('PyPhantomJS')
app.setOrganizationName('Umaclan Development')
app.setOrganizationDomain('www.umaclan.com')
app.setApplicationVersion(__version__)
args = parseArgs(app, sys.argv[1:])
phantom = Phantom(app, args)
do_action('Main')
if phantom.execute():
app.exec_()
return phantom.returnValue()
def handleStarted(self):
reply = self.sender()
if not reply:
return
if reply in self.m_started:
return
self.m_started.append(reply)
headers = []
for header in reply.rawHeaderList():
header = {
'name': str(header),
'value': str(reply.rawHeader(header))
}
headers.append(header)
data = {
'stage':
'start',
'id':
self.m_ids[reply],
'url':
reply.url().toString(),
'status':
reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
'statusText':
reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
'contentType':
reply.header(QNetworkRequest.ContentTypeHeader),
'bodySize':
reply.size(),
'redirectURL':
reply.header(QNetworkRequest.LocationHeader),
'headers':
headers,
'time':
QDateTime.currentDateTime()
}
do_action('NetworkAccessManagerHandleStarted')
self.resourceReceived.emit(data)
def __init__(self, parent, jsonFile):
super(Config, self).__init__(parent)
with codecs.open(jsonFile, encoding='utf-8') as fd:
json = fd.read()
self.settings = {
'cookiesFile': { 'mapping': 'cookies_file', 'default': None },
'debug': { 'mapping': 'debug', 'default': None },
'diskCache': { 'mapping': 'disk_cache', 'default': False },
'ignoreSslErrors': { 'mapping': 'ignore_ssl_errors', 'default': False },
'loadImages': { 'mapping': 'load_images', 'default': True },
'loadPlugins': { 'mapping': 'load_plugins', 'default': False },
'localToRemoteUrlAccessEnabled': { 'mapping': 'local_to_remote_url_access', 'default': False },
'maxDiskCacheSize': { 'mapping': 'max_disk_cache_size', 'default': -1 },
'outputEncoding': { 'mapping': 'output_encoding', 'default': 'System' },
'proxy': { 'mapping': 'proxy', 'default': None },
'scriptEncoding': { 'mapping': 'script_encoding', 'default': 'utf-8' },
'verbose': { 'mapping': 'verbose', 'default': False }
}
do_action('ConfigInit', self.settings)
# generate dynamic properties
for setting in self.settings:
self.setProperty(setting, self.settings[setting]['default'])
# now it's time to parse our JSON file
if not json.lstrip().startswith('{') or not json.rstrip().endswith('}'):
qWarning('Config file MUST be in JSON format!')
return
with QPyFile(':/configurator.js') as f:
configurator = f.readAll()
webPage = QWebPage(self)
# add config object
webPage.mainFrame().addToJavaScriptWindowObject('config', self)
# apply settings
webPage.mainFrame().evaluateJavaScript(configurator.replace('%1', json))
def main():
args = parseArgs(sys.argv[1:])
# register an alternative Message Handler
messageHandler = MessageHandler(args.verbose)
qInstallMsgHandler(messageHandler.process)
app = QApplication(sys.argv)
app.setWindowIcon(QIcon(':/resources/pyphantomjs-icon.png'))
app.setApplicationName('PyPhantomJS')
app.setOrganizationName('Umaclan Development')
app.setOrganizationDomain('www.umaclan.com')
app.setApplicationVersion(version)
phantom = Phantom(args, app)
do_action('Main', Bunch(locals()))
phantom.execute()
app.exec_()
sys.exit(phantom.returnValue())
class WebServerResponse(QObject):
def __init__(self, conn):
super(WebServerResponse, self).__init__()
self.setObjectName('WebServerResponse')
self.m_conn = conn
do_action('WebServerResponseInit')
@pyqtSlot(str, result=str)
def header(self, name):
return self.m_conn.m_headers.get(name, '')
@pyqtProperty('QVariantMap')
def headers(self):
return self.m_conn.m_headers
@headers.setter
def headers(self, headers):
self.m_conn.m_headers = CaseInsensitiveDict(headers)
@pyqtSlot(int)
@pyqtSlot(int, str)
def sendError(self, code, message=None):
self.m_conn.send_error(code, message)
self.m_conn.m_handleResponse = False
@pyqtSlot(str, str)
def setHeader(self, name, value):
self.m_conn.m_headers[name] = value
@pyqtProperty(int)
def statusCode(self):
return self.m_conn.m_statusCode
@statusCode.setter
def statusCode(self, code):
self.m_conn.m_statusCode = code
@pyqtSlot(str)
def write(self, body):
self.m_conn.m_wfile.write(body)
@pyqtSlot(int, 'QVariantMap')
def writeHead(self, code, headers):
self.m_conn.m_statusCode = code
self.m_conn.m_headers = CaseInsensitiveDict(headers)
do_action('WebServerResponse')
def handleFinished(self, reply):
headers = []
for header in reply.rawHeaderList():
header = {
'name': str(header),
'value': str(reply.rawHeader(header))
}
headers.append(header)
data = {
'stage':
'end',
'id':
self.m_ids[reply],
'url':
reply.url().toString(),
'status':
reply.attribute(QNetworkRequest.HttpStatusCodeAttribute),
'statusText':
reply.attribute(QNetworkRequest.HttpReasonPhraseAttribute),
'contentType':
reply.header(QNetworkRequest.ContentTypeHeader),
'redirectURL':
reply.header(QNetworkRequest.LocationHeader),
'headers':
headers,
'time':
QDateTime.currentDateTime()
}
del self.m_ids[reply]
if reply in self.m_started:
del self.m_started[self.m_started.index(reply)]
do_action('NetworkAccessManagerHandleFinished', Bunch(locals()))
self.resourceReceived.emit(data)
class WebPage(QWebPage):
def __init__(self, parent=None):
QWebPage.__init__(self, parent)
self.parent = parent
self.m_nextFileTag = ''
self.m_userAgent = QWebPage.userAgentForUrl(self, QUrl())
if self.parent.m_verbose:
self.currentFrame().urlChanged.connect(self.handleFrameUrlChanged)
self.linkClicked.connect(self.handleLinkClicked)
do_action('WebPageInit', Bunch(locals()))
def handleFrameUrlChanged(self, url):
qDebug('URL Changed: %s' % url.toString())
def handleLinkClicked(self, url):
qDebug('URL Clicked: %s' % url.toString())
def javaScriptAlert(self, webframe, msg):
print 'JavaScript alert: %s' % msg
def javaScriptConsoleMessage(self, message, lineNumber, sourceID):
if sourceID:
print '%s:%d %s' % (sourceID, lineNumber, message)
else:
print message
def shouldInterruptJavaScript(self):
QApplication.processEvents(QEventLoop.AllEvents, 42)
return False
def userAgentForUrl(self, url):
return self.m_userAgent
def chooseFile(self, webframe, suggestedFile):
if self.m_nextFileTag in self.parent.m_upload_file:
return self.parent.m_upload_file[self.m_nextFileTag]
return ''
do_action('WebPage', Bunch(locals()))
def __init__(self, args, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_loadStatus = self.m_state = ''
self.m_var = self.m_paperSize = self.m_loadScript_cache = {}
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_clipRect = QRect()
# setup the values from args
self.m_script = args.script.read()
self.m_scriptFile = args.script.name
self.m_scriptDir = os.path.dirname(args.script.name) + '/'
self.m_args = args.script_args
self.m_upload_file = args.upload_file
autoLoadImages = False if args.load_images == 'no' else True
pluginsEnabled = True if args.load_plugins == 'yes' else False
args.script.close()
do_action('PhantomInitPre', Bunch(locals()))
palette = self.m_page.palette()
palette.setBrush(QPalette.Base, Qt.transparent)
self.m_page.setPalette(palette)
if not args.proxy:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0],
int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
self.m_page.settings().setAttribute(QWebSettings.AutoLoadImages,
autoLoadImages)
self.m_page.settings().setAttribute(QWebSettings.PluginsEnabled,
pluginsEnabled)
self.m_page.settings().setAttribute(
QWebSettings.FrameFlatteningEnabled, True)
self.m_page.settings().setAttribute(
QWebSettings.OfflineStorageDatabaseEnabled, True)
self.m_page.settings().setAttribute(QWebSettings.LocalStorageEnabled,
True)
self.m_page.settings().setLocalStoragePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_page.settings().setOfflineStoragePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
# Ensure we have a document.body.
self.m_page.mainFrame().setHtml('<html><body></body></html>')
self.m_page.mainFrame().setScrollBarPolicy(Qt.Horizontal,
Qt.ScrollBarAlwaysOff)
self.m_page.mainFrame().setScrollBarPolicy(Qt.Vertical,
Qt.ScrollBarAlwaysOff)
m_netAccessMan = NetworkAccessManager(args.disk_cache,
args.ignore_ssl_errors, self)
self.m_page.setNetworkAccessManager(m_netAccessMan)
# inject our properties and slots into javascript
self.m_page.mainFrame().javaScriptWindowObjectCleared.connect(
self.inject)
self.m_page.loadFinished.connect(self.finish)
do_action('PhantomInitPost', Bunch(locals()))
class Phantom(QObject):
def __init__(self, args, parent=None):
QObject.__init__(self, parent)
# variable declarations
self.m_loadStatus = self.m_state = ''
self.m_var = self.m_paperSize = self.m_loadScript_cache = {}
self.m_verbose = args.verbose
self.m_page = WebPage(self)
self.m_clipRect = QRect()
# setup the values from args
self.m_script = args.script.read()
self.m_scriptFile = args.script.name
self.m_scriptDir = os.path.dirname(args.script.name) + '/'
self.m_args = args.script_args
self.m_upload_file = args.upload_file
autoLoadImages = False if args.load_images == 'no' else True
pluginsEnabled = True if args.load_plugins == 'yes' else False
args.script.close()
do_action('PhantomInitPre', Bunch(locals()))
palette = self.m_page.palette()
palette.setBrush(QPalette.Base, Qt.transparent)
self.m_page.setPalette(palette)
if not args.proxy:
QNetworkProxyFactory.setUseSystemConfiguration(True)
else:
proxy = QNetworkProxy(QNetworkProxy.HttpProxy, args.proxy[0],
int(args.proxy[1]))
QNetworkProxy.setApplicationProxy(proxy)
self.m_page.settings().setAttribute(QWebSettings.AutoLoadImages,
autoLoadImages)
self.m_page.settings().setAttribute(QWebSettings.PluginsEnabled,
pluginsEnabled)
self.m_page.settings().setAttribute(
QWebSettings.FrameFlatteningEnabled, True)
self.m_page.settings().setAttribute(
QWebSettings.OfflineStorageDatabaseEnabled, True)
self.m_page.settings().setAttribute(QWebSettings.LocalStorageEnabled,
True)
self.m_page.settings().setLocalStoragePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
self.m_page.settings().setOfflineStoragePath(
QDesktopServices.storageLocation(QDesktopServices.DataLocation))
# Ensure we have a document.body.
self.m_page.mainFrame().setHtml('<html><body></body></html>')
self.m_page.mainFrame().setScrollBarPolicy(Qt.Horizontal,
Qt.ScrollBarAlwaysOff)
self.m_page.mainFrame().setScrollBarPolicy(Qt.Vertical,
Qt.ScrollBarAlwaysOff)
m_netAccessMan = NetworkAccessManager(args.disk_cache,
args.ignore_ssl_errors, self)
self.m_page.setNetworkAccessManager(m_netAccessMan)
# inject our properties and slots into javascript
self.m_page.mainFrame().javaScriptWindowObjectCleared.connect(
self.inject)
self.m_page.loadFinished.connect(self.finish)
do_action('PhantomInitPost', Bunch(locals()))
def execute(self):
if self.m_script.startswith('#!'):
self.m_script = '//' + self.m_script
if self.m_scriptFile.lower().endswith('.coffee'):
coffee = CSConverter(self)
self.m_script = coffee.convert(self.m_script)
self.m_page.mainFrame().evaluateJavaScript(self.m_script)
def finish(self, success):
self.m_loadStatus = 'success' if success else 'fail'
self.m_page.mainFrame().evaluateJavaScript(self.m_script)
def inject(self):
self.m_page.mainFrame().addToJavaScriptWindowObject('phantom', self)
def renderPdf(self, fileName):
p = QPrinter()
p.setOutputFormat(QPrinter.PdfFormat)
p.setOutputFileName(fileName)
p.setResolution(pdf_dpi)
paperSize = self.m_paperSize
if not len(paperSize):
pageSize = QSize(self.m_page.mainFrame().contentsSize())
paperSize['width'] = str(pageSize.width()) + 'px'
paperSize['height'] = str(pageSize.height()) + 'px'
paperSize['border'] = '0px'
if paperSize.get('width') and paperSize.get('height'):
sizePt = QSizeF(ceil(self.stringToPointSize(paperSize['width'])),
ceil(self.stringToPointSize(paperSize['height'])))
p.setPaperSize(sizePt, QPrinter.Point)
elif 'format' in paperSize:
orientation = QPrinter.Landscape if paperSize.get(
'orientation') and paperSize['orientation'].lower(
) == 'landscape' else QPrinter.Portrait
orientation = QPrinter.Orientation(orientation)
p.setOrientation(orientation)
formats = {
'A3': QPrinter.A3,
'A4': QPrinter.A4,
'A5': QPrinter.A5,
'Legal': QPrinter.Legal,
'Letter': QPrinter.Letter,
'Tabloid': QPrinter.Tabloid
}
p.setPaperSize(QPrinter.A4) # fallback
for format, size in formats.items():
if format.lower() == paperSize['format'].lower():
p.setPaperSize(size)
break
else:
return False
border = floor(self.stringToPointSize(
paperSize['border'])) if paperSize.get('border') else 0
p.setPageMargins(border, border, border, border, QPrinter.Point)
self.m_page.mainFrame().print_(p)
return True
def returnValue(self):
return self.m_returnValue
def stringToPointSize(self, string):
units = (('mm', 72 / 25.4), ('cm', 72 / 2.54), ('in', 72.0),
('px', 72.0 / pdf_dpi / 2.54), ('', 72.0 / pdf_dpi / 2.54))
for unit, format in units:
if string.endswith(unit):
value = string.rstrip(unit)
return float(value) * format
return 0
##
# Properties and methods exposed to JavaScript
##
@pyqtProperty('QStringList')
def args(self):
return self.m_args
@pyqtProperty('QVariantMap')
def clipRect(self):
result = {
'width': self.m_clipRect.width(),
'height': self.m_clipRect.height(),
'top': self.m_clipRect.top(),
'left': self.m_clipRect.left()
}
return result
@clipRect.setter
def clipRect(self, size):
names = ('width', 'height', 'top', 'left')
for item in names:
try:
globals()[item] = int(size[item])
if globals()[item] < 0:
if item not in ('top', 'left'):
globals()[item] = 0
except KeyError:
globals()[item] = getattr(self.m_clipRect, item)()
self.m_clipRect = QRect(left, top, width, height)
@pyqtProperty(str)
def content(self):
return self.m_page.mainFrame().toHtml()
@content.setter
def content(self, content):
self.m_page.mainFrame().setHtml(content)
@pyqtSlot()
@pyqtSlot(int)
def exit(self, code=0):
self.m_returnValue = code
self.m_page.loadFinished.disconnect(self.finish)
QTimer.singleShot(0, qApp, SLOT('quit()'))
@pyqtProperty(str)
def loadStatus(self):
return self.m_loadStatus
@pyqtSlot(str, result=bool)
def loadScript(self, script):
if script in self.m_loadScript_cache:
self.m_page.mainFrame().evaluateJavaScript(
self.m_loadScript_cache[script])
return True
scriptFile = script
try:
script = codecs.open(self.m_scriptDir + script, encoding='utf-8')
script = script.read()
except IOError:
return False
if script.startswith('#!'):
script = '//' + script
if scriptFile.lower().endswith('.coffee'):
coffee = CSConverter(self)
script = coffee.convert(script)
self.m_loadScript_cache[scriptFile] = script
self.m_page.mainFrame().evaluateJavaScript(script)
return True
@pyqtSlot(str, name='open')
def open_(self, address):
qDebug('Opening address %s' % address)
self.m_page.triggerAction(QWebPage.Stop)
self.m_loadStatus = 'loading'
self.m_page.mainFrame().setUrl(QUrl(address))
@pyqtProperty('QVariantMap')
def paperSize(self):
return self.m_paperSize
@paperSize.setter
def paperSize(self, size):
self.m_paperSize = size
@pyqtSlot(str, result=bool)
def render(self, fileName):
fileInfo = QFileInfo(fileName)
path = QDir()
path.mkpath(fileInfo.absolutePath())
if fileName.lower().endswith('.pdf'):
return self.renderPdf(fileName)
viewportSize = QSize(self.m_page.viewportSize())
pageSize = QSize(self.m_page.mainFrame().contentsSize())
bufferSize = QSize()
if not self.m_clipRect.isEmpty():
bufferSize = self.m_clipRect.size()
else:
bufferSize = self.m_page.mainFrame().contentsSize()
if pageSize == '':
return False
image = QImage(bufferSize, QImage.Format_ARGB32)
image.fill(qRgba(255, 255, 255, 0))
p = QPainter(image)
p.setRenderHint(QPainter.Antialiasing, True)
p.setRenderHint(QPainter.TextAntialiasing, True)
p.setRenderHint(QPainter.SmoothPixmapTransform, True)
self.m_page.setViewportSize(pageSize)
if not self.m_clipRect.isEmpty():
p.translate(-self.m_clipRect.left(), -self.m_clipRect.top())
self.m_page.mainFrame().render(p, QRegion(self.m_clipRect))
else:
self.m_page.mainFrame().render(p)
p.end()
self.m_page.setViewportSize(viewportSize)
return image.save(fileName)
@pyqtSlot('QWebElement', str)
def setFormInputFile(self, el, fileTag):
self.m_page.m_nextFileTag = fileTag
el.evaluateJavaScript('''(function(target){
var evt = document.createEvent('MouseEvents');
evt.initMouseEvent("click", true, true, window,
0, 0, 0, 0, 0, false, false, false, false, 0, null);
target.dispatchEvent(evt);})(this);''')
@pyqtSlot(int)
def sleep(self, ms):
startTime = QTime.currentTime()
while True:
QApplication.processEvents(QEventLoop.AllEvents, 25)
if startTime.msecsTo(QTime.currentTime()) > ms:
break
usleep(0.005)
@pyqtProperty(str)
def state(self):
return self.m_state
@state.setter
def state(self, value):
self.m_state = value
@pyqtProperty(str)
def userAgent(self):
return self.m_page.m_userAgent
@userAgent.setter
def userAgent(self, ua):
self.m_page.m_userAgent = ua
@pyqtSlot(str, result='QVariant')
@pyqtSlot(int, result='QVariant')
@pyqtSlot(str, 'QVariant')
@pyqtSlot(int, 'QVariant')
def ctx(self, name, value=None):
if not value:
return self.m_var.get(name)
self.m_var[name] = value
@pyqtProperty('QVariantMap')
def version(self):
version = {
'major': version_major,
'minor': version_minor,
'patch': version_patch
}
return version
@pyqtProperty('QVariantMap')
def viewportSize(self):
size = self.m_page.viewportSize()
result = {'width': size.width(), 'height': size.height()}
return result
@viewportSize.setter
def viewportSize(self, size):
names = ('width', 'height')
for item in names:
try:
globals()[item] = int(size[item])
if globals()[item] < 0:
globals()[item] = 0
except KeyError:
globals()[item] = getattr(self.m_page.viewportSize(), item)()
self.m_page.setViewportSize(QSize(width, height))
do_action('Phantom', Bunch(locals()))
