class StatusArea(QWidget):
def __init__(self, parent=None):
super(StatusArea, self).__init__(parent)
self.setStyleSheet('StatusArea {background: yellow}')
self.msg = QLabel(self)
self.file = QLabel(self)
self.progress = QProgressBar(self)
self.msg.setFont(View.labelsFont())
self.file.setFont(View.editsFont())
self.progress.setMaximum(100)
self.progress.setMinimum(0)
self.progress.setTextVisible(False)
self.progress.setStyleSheet("""
QProgressBar {
border: 2px solid grey;
border-radius: 5px;
width: 60px;
height: 10px;
}
QProgressBar::chunk {
background-color: #05B8CC;
width: 5px;
}""")
layout = QHBoxLayout()
layout.addWidget(self.msg, 0, Qt.AlignLeft)
layout.addWidget(self.file, 0, Qt.AlignLeft)
layout.addWidget(self.progress, 0, Qt.AlignRight)
self.setLayout(layout)
@Slot(str)
@Slot(str, str, str)
def setMessage(self, msg, file='', progress=None):
if not progress:
self.progress.hide()
self.progress.setValue(0)
else:
self.progress.setValue(progress)
self.progress.show()
self.msg.setText(msg)
self.file.setText(file)
def paintEvent(self, event):
p = QPainter()
p.begin(self)
p.fillRect(self.rect(), QBrush(QColor(240, 200, 0)))
p.end()
class StatusArea(QWidget):
def __init__(self, parent=None):
super(StatusArea, self).__init__(parent)
self.setStyleSheet('StatusArea {background: yellow}')
self.msg = QLabel(self)
self.file = QLabel(self)
self.progress = QProgressBar(self)
self.msg.setFont(View.labelsFont())
self.file.setFont(View.editsFont())
self.progress.setMaximum(100)
self.progress.setMinimum(0)
self.progress.setTextVisible(False)
self.progress.setStyleSheet("""
QProgressBar {
border: 2px solid grey;
border-radius: 5px;
width: 60px;
height: 10px;
}
QProgressBar::chunk {
background-color: #05B8CC;
width: 5px;
}""")
layout = QHBoxLayout()
layout.addWidget(self.msg, 0, Qt.AlignLeft)
layout.addWidget(self.file, 0, Qt.AlignLeft)
layout.addWidget(self.progress, 0, Qt.AlignRight)
self.setLayout(layout)
@Slot(str)
@Slot(str, str, str)
def setMessage(self, msg, file='', progress=None):
if not progress:
self.progress.hide()
self.progress.setValue(0)
else:
self.progress.setValue(progress)
self.progress.show()
self.msg.setText(msg)
self.file.setText(file)
def paintEvent(self, event):
p = QPainter()
p.begin(self)
p.fillRect(self.rect(), QBrush(QColor(240, 200, 0)))
p.end()
class ControlledProgressDialog(QtGui.QProgressDialog):
"""
QProgressDialog controlled by pipe
"""
def __init__(self,inQueue,outQueue,*args,**kwargs):
QtGui.QProgressDialog.__init__(self,*args,**kwargs)
self.progress = QProgressBar(self)
self.progress.setFormat("%p%")
self.setBar(self.progress)
self.inQueue = inQueue
self.outQueue = outQueue
self.timer = QtCore.QTimer(self)
self.timer.timeout.connect(self.dispatcher)
self.timer.start(50)
self.center()
def setTextVisible(self,visible):
self.progress.setTextVisible(visible)
def center(self):
screen = QtGui.QDesktopWidget().screenGeometry()
size = self.geometry()
self.move((screen.width()-size.width())/2, (screen.height()-size.height())/2)
@QtCore.Slot()
def dispatcher(self):
"""
Dispatcher called by 50ms
"""
while not self.inQueue.empty():
cmd,args,ret = self.inQueue.get()
if cmd == "quit":
self.timer.stop()
self.close()
else:
res = getattr(self,cmd)(*args)
if ret:
self.outQueue.put(res)
self.center()
class SpiderTab(QWidget):
"""Has handlers for spider data and events. It houses the results table of the
spider, controls for the spider and progress indication
It implicitly conforms to IEventHandler interface"""
TIMER_CHECK_INTERVAL = 3000
favicon_received = Signal(str) # send the url or path to the handler, which should be the tab widget
stop_spider_signal = Signal(int)
became_current = Signal(bool) # tell the table it has become active. it's an interesting property for producers!
def __init__(self, parent=None, **kwargs):
super(SpiderTab, self).__init__(parent)
self._event_queue = None
self._data_queue = None
self._engine = None
self._favicon_received = False
self._spider_id = None
self._item_count = 0
self.setContextMenuPolicy(Qt.ContextMenuPolicy.DefaultContextMenu)
self.initInterface(kwargs)
self._context_menu = None
self._setupContextMenu()
self.became_current.connect(self._set_table_activity)
self._queue_check_timer = QTimer()
self._queue_check_timer.setInterval(self.TIMER_CHECK_INTERVAL)
self._queue_check_timer.timeout.connect(self._checkQueues)
self._queue_check_timer.start()
def initInterface(self, kwargs):
layout = QGridLayout()
self._data_table = SearchTable(name=kwargs.get("name"))
self._progress_spider = QProgressBar()
self._label_count = QLabel(self.tr("0 items scraped"))
# make it a busy indicator. you don't know when it'll finish
self._progress_spider.setMinimum(0); self._progress_spider.setMaximum(0)
self._progress_spider.setTextVisible(False)
self._btn_stop_spider = QPushButton(self.tr("Stop Spider"))
self._btn_stop_spider.clicked.connect(self.stop_spider)
row = 0; col = 0;
layout.addWidget(self._data_table, row, col, 1, 4)
row += 1;
layout.addWidget(self._progress_spider, row, col, 1, 1)
col += 1
layout.addWidget(self._label_count, row, col, 1, 2)
col += 2
layout.addWidget(self._btn_stop_spider, row, col, 1, 1)
self.setLayout(layout)
def _setupContextMenu(self):
from visualscrape.lib.data import ActionStore
self._context_menu = QMenu(self)
# get the export action from the action store
action_store = ActionStore.get_instance()
for action in action_store:
if action.get_name() == "export":
export_action = action
break
self._context_menu.addAction(export_action)
def export_table(self):
export_dialog = ExportDialog()
export_dialog.exec_()
export_info = export_dialog.data()
if export_info:
data = self._data_table.get_visible_data()
FileExporter.export(data, self._data_table.name.lower(), export_info.location, export_info.format)
def set_event_queue(self, eq):
self._event_queue = eq
def set_data_queue(self, dq):
self._data_queue = dq
def stop_spider(self):
if self._spider_id is None: # do not stop the the spider before receiving data
pass
else:
if self._queue_check_timer.isActive():
confirm_stop = QMessageBox(self)
confirm_stop.setIcon(QMessageBox.Warning)
confirm_stop.setStandardButtons(QMessageBox.Yes | QMessageBox.No)
confirm_stop.setText(self.tr("Scraping process still running"))
confirm_stop.setDetailedText(self.tr("Are you sure you want to stop it?"))
confirm_stop.setWindowTitle(self.tr("Spider still running"))
ret = confirm_stop.exec_()
if ret == QMessageBox.Yes:
self.stop_spider_signal.emit(self._spider_id)
return True
else: return False # I won't whip you if you stop it accidentally
else: return True # already over
def configure_searchlineedit(self, lineEdit):
self._data_table.configure_search_lineedit(lineEdit)
def _checkQueues(self):
while not self._event_queue.empty():
event = self._event_queue.get(block=False, timeout=0)
if isinstance(event, SpiderClosed):
self._queue_check_timer.stop()
self._progress_spider.setMinimum(0)
self._progress_spider.setMaximum(100)
self._progress_spider.setValue(100)
self._btn_stop_spider.setEnabled(False)
while not self._data_queue.empty():
item = self._data_queue.get(block=False, timeout=0)
if not self._favicon_received: # the first item on the data queue should be the favicon
favicon_data = item["images"][0]
self.favicon_received.emit(favicon_data["path"]) # note that icons are not guaranteed to have a path. Not everybody wants to save images
self._favicon_received = True
self._spider_id = item["_id"]
else:
item.pop("_id") # the table has nothing to do with spider ids
self._data_table.addItem(item)
self._item_count += 1
self._label_count.setText(self.tr("{0:n} items scraped".format(self._item_count)))
def _set_table_activity(self, state):
self._data_table.set_active(state)
