def parseArgs(app, args): # Handle all command-line options p = argParser() arg_data = p.parse_known_args(args) args = arg_data[0] args.script_args = arg_data[1] # convert script args to unicode for i, arg in enumerate(args.script_args): args.script_args[i] = unicode(arg, 'utf-8') # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) file_check = (args.cookies_file, args.config) for file_ in file_check: if file_ is not None and not os.path.exists(file_): sys.exit("No such file or directory: '%s'" % file_) if args.config: config = Config(app, args.config) # apply settings for setting in config.settings: setattr(args, config.settings[setting]['mapping'], config.property(setting)) split_check = ( (args.proxy, 'proxy'), ) for arg, name in split_check: if arg: item = arg.split(':') if len(item) < 2 or not len(item[1]): p.print_help() sys.exit(1) setattr(args, name, item) if args.proxy is not None: if args.proxy_type == 'socks5': args.proxy_type = QNetworkProxy.Socks5Proxy do_action('ParseArgs', args) if args.debug: debug(args.debug) # verbose flag got changed on us, so we reload the flag if messageHandler.verbose != args.verbose: messageHandler.verbose = args.verbose if args.script is None: p.print_help() sys.exit(1) if not os.path.exists(args.script): sys.exit("No such file or directory: '%s'" % args.script) return args
def parseArgs(app, args): # Handle all command-line options p = argParser() arg_data = p.parse_known_args(args) args = arg_data[0] args.script_args = arg_data[1] # convert script args to unicode for i, arg in enumerate(args.script_args): args.script_args[i] = unicode(arg, 'utf-8') # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) file_check = (args.cookies_file, args.config) for file_ in file_check: if file_ is not None and not os.path.exists(file_): sys.exit("No such file or directory: '%s'" % file_) if args.config: config = Config(app, args.config) # apply settings for setting in config.settings: setattr(args, config.settings[setting]['mapping'], config.property(setting)) split_check = ((args.proxy, 'proxy'), ) for arg, name in split_check: if arg: item = arg.split(':') if len(item) < 2 or not len(item[1]): p.print_help() sys.exit(1) setattr(args, name, item) if args.proxy is not None: if args.proxy_type == 'socks5': args.proxy_type = QNetworkProxy.Socks5Proxy do_action('ParseArgs', args) if args.debug: debug(args.debug) # verbose flag got changed on us, so we reload the flag if messageHandler.verbose != args.verbose: messageHandler.verbose = args.verbose if args.script is None: p.print_help() sys.exit(1) if not os.path.exists(args.script): sys.exit("No such file or directory: '%s'" % args.script) return args
def parseArgs(app, args): # Handle all command-line options p = argParser() arg_data = p.parse_known_args(args) args = arg_data[0] args.script_args = arg_data[1] args.disk_cache = False if args.disk_cache == 'no' else True args.ignore_ssl_errors = False if args.ignore_ssl_errors == 'no' else True args.load_images = True if args.load_images == 'yes' else False args.load_plugins = False if args.load_plugins == 'no' else True args.local_access_remote = False if args.local_access_remote == 'no' else True # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) file_check = (args.cookies, args.config) for file_ in file_check: if file_ is not None and not os.path.exists(file_): sys.exit("No such file or directory: '%s'" % file_) if args.config: config = Config(app, args.config) # apply settings for setting in config.settings: setattr(args, config.settings[setting]['mapping'], config.property(setting)) # special case for verbose arg, which will need to be re-applied if setting == 'verbose': messageHandler.verbose = args.verbose split_check = ( (args.proxy, 'proxy'), (args.auth, 'auth') ) for arg, name in split_check: if arg: item = arg.split(':') if len(item) < 2 or not len(item[1]): p.print_help() sys.exit(1) setattr(args, name, item) do_action('ParseArgs') if args.script is None: p.print_help() sys.exit(1) if not os.path.exists(args.script): sys.exit("No such file or directory: '%s'" % args.script) return args
def __init__(self, name, settings): super(Application, self).__init__([]) self.name = name self.settings = settings self.web_view = QWebView() self._exit_timer = QTimer(self) self._exit_timer.setSingleShot(True) self._exit_timer.setInterval(1000) self._expects = [] self._expects_if_timeout = [] self._expects_timer = QTimer(self) self._expects_timer.setSingleShot(True) self._expects_timer.timeout.connect(self._on_expects_timeout) self._frame_data = {} self._frame_data_lock = Lock() self._frame_timer = QTimer(self) self._frame_timer.start(3000) self._queue = Queue() self._trigger_delay_timer = QTimer(self) self._trigger_delay_timer.setSingleShot(True) self._visible = int(self.settings['application.visible']) self.web_page = WebPage(self.web_view) self.web_page.log_event.connect(self.log_event) self.web_page.frameCreated.connect(self._on_frame_created) self._on_frame_created(self.web_page.mainFrame()) #self.web_page.networkAccessManager().finished.connect( # self._on_http_response) self.web_view.setPage(self.web_page) st = self.web_page.settings() st.setAttribute(st.AutoLoadImages, int(self.settings['application.settings.load_images'])) st.setAttribute( st.JavaEnabled, int(self.settings['application.settings.java_enabled'])) st.setAttribute( st.PluginsEnabled, int(self.settings['application.settings.plugins_enabled'])) self.clear_handlers() # redirect qt related messages try: qInstallMessageHandler(self._pyqt5_null_message_handler) except NameError: qInstallMsgHandler(self._pyqt4_null_message_handler)
def __init__(self, user_agent=default_user_agent, wait_timeout=20, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), cache_dir='/tmp/ghost.py', cache_size=10, plugin_path=[ '/usr/lib/mozilla/plugins', ], share_cookies=True, share_cache=True, qt_debug=False): self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.viewport_size = viewport_size self.log_level = log_level self.display = display self.share_cookies = share_cookies self.share_cache = share_cache self.cache_dir = cache_dir self.cache_size = cache_size self.network_managers = [] self.current_page = None self._pages = [] if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) for p in plugin_path: Ghost._app.addLibraryPath(p) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) logger.setLevel(log_level)
def main(): args = parseArgs(sys.argv[1:]) # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) app = QApplication(sys.argv) app.setWindowIcon(QIcon(':/resources/pyphantomjs-icon.png')) app.setApplicationName('PyPhantomJS') app.setOrganizationName('Umaclan Development') app.setOrganizationDomain('www.umaclan.com') app.setApplicationVersion(version) phantom = Phantom(args, app) phantom.execute() app.exec_() sys.exit(phantom.returnValue())
def __init__(self, user_agent=default_user_agent, wait_timeout=20, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), cache_dir='/tmp/ghost.py', cache_size=10, plugin_path=['/usr/lib/mozilla/plugins',], share_cookies=True, share_cache=True, qt_debug=False): self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.viewport_size = viewport_size self.log_level = log_level self.display = display self.share_cookies = share_cookies self.share_cache = share_cache self.cache_dir = cache_dir self.cache_size = cache_size self.network_managers = [] self.current_page = None self._pages = [] if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Exception('Xvfb is required to a ghost run oustside ' +\ 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) for p in plugin_path: Ghost._app.addLibraryPath(p) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) logger.setLevel(log_level)
def main(): args = parseArgs(sys.argv[1:]) # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) app = QApplication(sys.argv) app.setWindowIcon(QIcon(":/resources/pyphantomjs-icon.png")) app.setApplicationName("PyPhantomJS") app.setOrganizationName("Umaclan Development") app.setOrganizationDomain("www.umaclan.com") app.setApplicationVersion(version) phantom = Phantom(args) do_action("Main", Bunch(locals())) if phantom.execute(): app.exec_() return phantom.returnValue()
from PyQt4.QtCore import QString, qInstallMsgHandler, qFatal from PyQt4.QtGui import * # make keyboard interrupt quit program import signal signal.signal(signal.SIGINT, signal.SIG_DFL) if __name__ == "__main__": # Handle all command-line options p = argParser() args = p.parse_args() # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) if args.upload_file: item_buffer = {} for i in range(len(args.upload_file)): item = args.upload_file[i].split("=") if len(item) < 2 or not len(item[1]): if len(item_buffer) == 0: p.print_help() sys.exit(1) args.script = args.upload_file[i:] break item_buffer[QString(item[0])] = QString(item[1]) for tag in item_buffer: if not os.path.exists(item_buffer[tag]): qFatal("No such file or directory: '%s'" % item_buffer[tag])
from PyQt4.QtGui import QIcon, QApplication # make keyboard interrupt quit program import signal signal.signal(signal.SIGINT, signal.SIG_DFL) if __name__ == '__main__': # Handle all command-line options p = argParser() arg_data = p.parse_known_args(sys.argv[1:]) args = arg_data[0] args.script_args = arg_data[1] # register an alternative Message Handler messageHandler = MessageHandler(args.verbose) qInstallMsgHandler(messageHandler.process) if args.upload_file: item_buffer = {} for i in range(len(args.upload_file)): item = args.upload_file[i].split('=') if len(item) < 2 or not len(item[1]): if len(item_buffer) == 0: p.print_help() sys.exit(1) # this is a bug workaround for argparse. # if you call parse_known_args, and you # have an --option script arg, the args # get jumbled up, and it's inconsistent #
import sys from PyQt4.QtGui import QApplication from PyQt4.QtCore import QUrl, qInstallMsgHandler from PyQt4.QtWebKit import QWebPage, QWebSettings # comment this line to show QT warnings/errors: qInstallMsgHandler(lambda *args: None) class Render(QWebPage): def __init__(self, cb, dump_file): self.app = QApplication(sys.argv) QWebPage.__init__(self) self.mainFrame().loadFinished.connect(self._loadFinished) self.cb = cb self.html_dump = dump_file self.settings().setAttribute(QWebSettings.AutoLoadImages, False) def crawl(self, url): print('Downloading', url) self.mainFrame().load(QUrl(url)) self.app.exec_() def _loadFinished(self, result): frame = self.mainFrame() url = str(frame.url().toString()) html = frame.toHtml() self.cb(url, html) with open(self.html_dump, 'w', encoding='utf-8') as dump: dump.write(html) self.app.quit()
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=[ '/usr/lib/mozilla/plugins', ], download_images=True, qt_debug=False): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Error('Xvfb is required to a ghost run outside ' + 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.popup_messages = [] self.page = GhostWebPage(Ghost._app, self) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute( QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: class MyQWebView(QtWebKit.QWebView): def sizeHint(self): return QSize(*viewport_size) self.webview = MyQWebView() if plugins_enabled: self.webview.settings().setAttribute( QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute( QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() else: self.webview = None
def __call__(self, msgType, wtf, msg): #print('1)', msgType) #print('2)', msg) #print('3)', zz) levels = { QtDebugMsg: 'debug', QtWarningMsg: 'warn', QtCriticalMsg: 'critical', QtFatalMsg: 'fatal', } getattr(self.logger, levels[msgType])(msg) qt_logger = configure_logger('qt', 'QT', logging.DEBUG, logging.StreamHandler()) qInstallMsgHandler(QTMessageProxy(qt_logger)) class WKitWebView(QWebView): def setApplication(self, app): self.app = app def closeEvent(self, event): self.app.quit() def sizeHint(self): viewport_size = (800, 600) return QSize(*viewport_size) class WKitWebPage(QWebPage):
try: from queue import Empty, Queue except ImportError: from Queue import Empty, Queue from uuid import uuid4 from core.helpers import flatten_settings_definition, make_list from core.proxy import FrameData, Proxy def null_message_handler(*args, **kwargs): pass # disable qt related messages qInstallMsgHandler(null_message_handler) class WebPage(QWebPage): """ QWebPage that prints Javascript errors to logger. Adapted from http://www.tylerlesmann.com/2009/oct/01/web-scraping-pyqt4/ """ onLog = pyqtSignal(int, str) def javaScriptConsoleMessage(self, message, lineNumber, sourceID): self.onLog.emit( DEBUG, 'Javascript:%s:%s: %s' % (sourceID, lineNumber, message))
def __init__(self, user_agent=default_user_agent, wait_timeout=8, wait_callback=None, log_level=logging.WARNING, display=False, viewport_size=(800, 600), ignore_ssl_errors=True, cache_dir=os.path.join(tempfile.gettempdir(), "ghost.py"), plugins_enabled=False, java_enabled=False, plugin_path=['/usr/lib/mozilla/plugins',], download_images=True, qt_debug=False): self.http_resources = [] self.user_agent = user_agent self.wait_timeout = wait_timeout self.wait_callback = wait_callback self.ignore_ssl_errors = ignore_ssl_errors self.loaded = True if not sys.platform.startswith('win') and not 'DISPLAY' in os.environ\ and not hasattr(Ghost, 'xvfb'): try: os.environ['DISPLAY'] = ':99' Ghost.xvfb = subprocess.Popen(['Xvfb', ':99']) except OSError: raise Error('Xvfb is required to a ghost run outside ' + 'an X instance') self.display = display if not Ghost._app: Ghost._app = QApplication.instance() or QApplication(['ghost']) qInstallMsgHandler(QTMessageProxy(qt_debug)) if plugin_path: for p in plugin_path: Ghost._app.addLibraryPath(p) self.popup_messages = [] self.page = GhostWebPage(Ghost._app, self) QtWebKit.QWebSettings.setMaximumPagesInCache(0) QtWebKit.QWebSettings.setObjectCacheCapacities(0, 0, 0) QtWebKit.QWebSettings.globalSettings().setAttribute(QtWebKit.QWebSettings.LocalStorageEnabled, True) self.page.setForwardUnsupportedContent(True) self.page.settings().setAttribute(QtWebKit.QWebSettings.AutoLoadImages, download_images) self.page.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, plugins_enabled) self.page.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, java_enabled) self.set_viewport_size(*viewport_size) # Page signals self.page.loadFinished.connect(self._page_loaded) self.page.loadStarted.connect(self._page_load_started) self.page.unsupportedContent.connect(self._unsupported_content) self.manager = self.page.networkAccessManager() self.manager.finished.connect(self._request_ended) self.manager.sslErrors.connect(self._on_manager_ssl_errors) # Cache self.cache = QNetworkDiskCache() self.cache.setCacheDirectory(cache_dir) self.manager.setCache(self.cache) # Cookie jar self.cookie_jar = QNetworkCookieJar() self.manager.setCookieJar(self.cookie_jar) # User Agent self.page.setUserAgent(self.user_agent) self.page.networkAccessManager().authenticationRequired\ .connect(self._authenticate) self.page.networkAccessManager().proxyAuthenticationRequired\ .connect(self._authenticate) self.main_frame = self.page.mainFrame() logger.setLevel(log_level) if self.display: class MyQWebView(QtWebKit.QWebView): def sizeHint(self): return QSize(*viewport_size) self.webview = MyQWebView() if plugins_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.PluginsEnabled, True) if java_enabled: self.webview.settings().setAttribute(QtWebKit.QWebSettings.JavaEnabled, True) self.webview.setPage(self.page) self.webview.show() else: self.webview = None
from PyQt4.QtCore import * from PyQt4.QtGui import * from PyQt4.QtWebKit import * from PyQt4.QtNetwork import * from PyQt4.QtCore import QUrl #prevent qt messages from the Terminal screen from PyQt4.QtCore import qInstallMsgHandler from PyQt4.Qt import QtMsgType def myQtMsgHandler(msg_type, msg_string): pass qInstallMsgHandler(myQtMsgHandler) from scrapex import common, http, agent app = QApplication(sys.argv) class WebView(QWebView): def __init__(self, show=False, timeout=30, image=False, js=True, **options): QWebView.__init__(self)