def __init__(self, profile, main_window):
QWebEngineView.__init__(self, main_window)
self.middle_click_soon = 0
self.needs_fake_focus = False
self.setAttribute(
Qt.WA_DeleteOnClose
) # needed otherwise object is not deleted on close which means, it keeps running
self.setMinimumWidth(300)
self.follow_link_pending = None
self.setFocusPolicy(Qt.ClickFocus | Qt.WheelFocus)
self.pending_unserialize = None
self.main_window = main_window
self.create_page(profile)
self.view_id = next(view_id)
self.iconChanged.connect(self.on_icon_changed,
type=Qt.QueuedConnection)
self.set_editable_text_in_gui_thread.connect(self.set_editable_text,
type=Qt.QueuedConnection)
self.loadStarted.connect(self.load_started)
self.loadFinished.connect(self.load_finished)
self._page.linkHovered.connect(self.link_hovered.emit)
self._page.windowCloseRequested.connect(
lambda: self.window_close_requested.emit(self))
self.popup = Popup(self)
self._page.featurePermissionRequested.connect(
self.feature_permission_requested)
self._page.featurePermissionRequestCanceled.connect(
self.feature_permission_request_canceled)
self._page.fullScreenRequested.connect(self.full_screen_requested)
self.feature_permission_map = {}
self.text_input_focused = False
self._force_passthrough = False
self.titleChanged.connect(self.on_title_change)
self.renderProcessTerminated.connect(self.render_process_terminated)
self.callback_on_save_edit_text_node = None
class MainWindow(QWidget):
def __init__(self):
super().__init__()
self.setWindowTitle('...')
self.url_le = QLineEdit('http://qt-project.org/')
self.go_pb = QPushButton('Go')
self.go_pb.clicked.connect(self._on_load_url)
url_layout = QHBoxLayout()
url_layout.addWidget(self.url_le)
url_layout.addWidget(self.go_pb)
self.view = QWebEngineView()
self.view.urlChanged.connect(self._on_url_changed)
self.view.titleChanged.connect(self.setWindowTitle)
main_layout = QVBoxLayout()
main_layout.addLayout(url_layout)
main_layout.addWidget(self.view)
self.setLayout(main_layout)
def _on_load_url(self):
self.view.load(QUrl(self.url_le.text()))
def _on_url_changed(self, url: QUrl):
self.url_le.setText(url.toString())
def __init__(self, profile, parent=None):
QWebEngineView.__init__(self, parent)
self._page = Page(profile, self)
self._page.set_title.connect(self.set_title.emit)
self._page.refresh_all.connect(self.refresh_all.emit)
self.titleChanged.connect(self._page.check_for_messages_from_js,
type=Qt.QueuedConnection)
self.setPage(self._page)
self.load(
QUrl.fromLocalFile(
os.path.join(os.path.dirname(os.path.abspath(__file__)),
'index.html')))
self.renderProcessTerminated.connect(self.render_process_terminated)
def setup_ui(self):
self.setObjectName('MainWindow')
self.resize(1000, 600)
self.web_view = QWebEngineView(self)
self.web_view.resize(1000, 600)
self.setCentralWidget(self.web_view)
self.web_profile = QWebEngineProfile(self.web_view)
self.web_profile.settings().setAttribute(
QWebEngineSettings.LocalStorageEnabled, True)
self.web_page = WebPage(self.web_profile, self)
self.web_page.application = self
self.web_view.setPage(self.web_page)
self.chanel = QWebChannel(self.web_page)
self.web_page.setWebChannel(self.chanel)
self.chanel.registerObject('bridge', self.web_page)
self.chanel.registerObject('spider', self.spider_object)
self.chanel.registerObject('urls', self.urls_object)
self.set_page()
def __init__(self):
super().__init__()
self.setWindowTitle('...')
self.url_le = QLineEdit('http://qt-project.org/')
self.go_pb = QPushButton('Go')
self.go_pb.clicked.connect(self._on_load_url)
url_layout = QHBoxLayout()
url_layout.addWidget(self.url_le)
url_layout.addWidget(self.go_pb)
self.view = QWebEngineView()
self.view.urlChanged.connect(self._on_url_changed)
self.view.titleChanged.connect(self.setWindowTitle)
main_layout = QVBoxLayout()
main_layout.addLayout(url_layout)
main_layout.addWidget(self.view)
self.setLayout(main_layout)
def moveEvent(self, ev):
self.moved.emit()
return QWebEngineView.moveEvent(self, ev)
def resizeEvent(self, ev):
self.resized.emit()
return QWebEngineView.resizeEvent(self, ev)
def sizeHint(self):
return default_size_hint(QWebEngineView.sizeHint(self))
def __init__(self, parent=None):
QWebEngineView.__init__(self, parent)
class Application(QMainWindow):
available_browsers = []
current_browsers = []
loaded_browsers = {}
force_compile = True
storage = None
visited = []
spider_running = False
spider = None
"""
:type: ShumSpider
"""
def __init__(self, parent=None):
super(Application, self).__init__(parent)
self.available_browsers = webbrowser._browsers
self.storage = Storage()
self.spider = self.create_spider()
self.spider.task_interval = self.storage.options.get(
'task_interval', 1)
self.spider_process = Process(target=self.spider.run)
self.spider_object = SpiderObject(self)
self.urls_object = UrlsObject(self)
self.force_compile = True
self.queue = Queue()
self.setup_ui()
def setup_ui(self):
self.setObjectName('MainWindow')
self.resize(1000, 600)
self.web_view = QWebEngineView(self)
self.web_view.resize(1000, 600)
self.setCentralWidget(self.web_view)
self.web_profile = QWebEngineProfile(self.web_view)
self.web_profile.settings().setAttribute(
QWebEngineSettings.LocalStorageEnabled, True)
self.web_page = WebPage(self.web_profile, self)
self.web_page.application = self
self.web_view.setPage(self.web_page)
self.chanel = QWebChannel(self.web_page)
self.web_page.setWebChannel(self.chanel)
self.chanel.registerObject('bridge', self.web_page)
self.chanel.registerObject('spider', self.spider_object)
self.chanel.registerObject('urls', self.urls_object)
self.set_page()
def set_page(self):
if self.force_compile or not exists(realpath('./data/compiled/index.html')) or \
getmtime(realpath('./interface/index.html')) > getmtime(realpath('./data/compiled/index.html')):
self._compile_template('index.html', './data/compiled/index.html')
qFile_page = QFile(realpath('./data/compiled/index.html'))
self.web_view.load(
QUrl.fromLocalFile(QFileInfo(qFile_page).absoluteFilePath()))
self.web_page.selectionChanged.emit()
@staticmethod
def _compile_template(template, output):
env = Environment(loader=FileSystemLoader('./interface'))
template = env.get_template(template)
file = open(realpath(output), 'w')
file.write(template.render(basepath='../../interface'))
file.close()
def get_browser(self, name):
"""
:param name:
:return: Browser object
:rtype: ChromeBrowser
"""
if name == 'chrome' or name == 'google-chrome':
if 'chrome' not in self.loaded_browsers:
self.loaded_browsers['chrome'] = ChromeBrowser(None)
return self.loaded_browsers['chrome']
elif name == 'firefox':
if 'firefox' not in self.loaded_browsers:
self.loaded_browsers['firefox'] = FirefoxBrowser(
webbrowser.get(name))
return self.loaded_browsers['firefox']
elif name == 'opera':
if 'opera' not in self.loaded_browsers:
self.loaded_browsers['opera'] = OperaBrowser(
webbrowser.get(name))
return self.loaded_browsers['opera']
else:
raise BrowserNotSupport(name)
def load_browser_history(self, browser):
browser = self.get_browser(browser)
history_generator = browser.get_history()
loaded_elements = 0
for element in history_generator:
url = self.storage.urls.create()
url.address = element.url
url.headers = element.headers
url.cookies = []
url.date_add = time()
url.last_visit = 0
url.request_type = 'GET'
url.is_active = 1
url.add_by = 'History import'
if self.is_valid_url(url):
if url.save():
loaded_elements += 1
else:
print('Cant save history element ' + element.url)
else:
del url
self.storage.connection.commit()
return loaded_elements
def run_spider(self, count=None):
if not self.spider_running:
self.queue = Queue()
self.spider_process = Process(target=self._start_spider,
args=(count, self.queue))
self.spider_running = True
self.spider_process.start()
logger = logging.getLogger('shum.app')
logger.debug('Spider running')
def _start_spider(self, count, queue):
self.spider = self.create_spider()
self.spider.application = self
self.spider.history_args = {'limit': {'count': count}}
self.spider.run(queue)
def stop_spider(self):
if self.spider_running:
self.spider_running = False
self.spider_process.terminate()
self.spider.stop()
logger = logging.getLogger('shum.app')
logger.debug('Spider stopped')
@staticmethod
def create_spider():
return ShumSpider(thread_number=10, transport='threaded')
def is_valid_url(self, url):
available_schemas = self.storage.options.get('available_schemas', [])
blocked_domains = self.storage.options.get('blocked_domains', [])
if url._parsed.scheme in available_schemas:
for blocked_domain in blocked_domains:
if match(blocked_domain, url._parsed.netloc) is not None:
return False
return True
return False
def moveEvent(self, ev):
self.moved.emit()
return QWebEngineView.moveEvent(self, ev)
def resizeEvent(self, ev):
self.resized.emit()
return QWebEngineView.resizeEvent(self, ev)
def event(self, event):
if event.type() == event.ChildPolished:
child = event.child()
if 'HostView' in child.metaObject().className():
self.host_widget = child
return QWebEngineView.event(self, event)
def __init__(self, profile, main_window):
QWebEngineView.__init__(self, main_window)
self.middle_click_soon = 0
self.setAttribute(Qt.WA_DeleteOnClose) # needed otherwise object is not deleted on close which means, it keeps running
self.setMinimumWidth(300)
self.follow_link_pending = None
self.setFocusPolicy(Qt.ClickFocus | Qt.WheelFocus)
self.pending_unserialize = None
self.main_window = main_window
self.create_page(profile)
self.view_id = next(view_id)
self.iconChanged.connect(self.on_icon_changed, type=Qt.QueuedConnection)
self.set_editable_text_in_gui_thread.connect(self.set_editable_text, type=Qt.QueuedConnection)
self.loadStarted.connect(self.load_started)
self.loadFinished.connect(self.load_finished)
self._page.linkHovered.connect(self.link_hovered.emit)
self._page.windowCloseRequested.connect(lambda: self.window_close_requested.emit(self))
self.popup = Popup(self)
self._page.featurePermissionRequested.connect(self.feature_permission_requested)
self._page.featurePermissionRequestCanceled.connect(self.feature_permission_request_canceled)
self._page.fullScreenRequested.connect(self.full_screen_requested)
self.feature_permission_map = {}
self.text_input_focused = False
self._force_passthrough = False
self.titleChanged.connect(self.on_title_change)
self.renderProcessTerminated.connect(self.render_process_terminated)