def init_form(self):
vlayout = QVBoxLayout()
if _api.USED_API == _api.QT_API_PYQT5:
vlayout.setContentsMargins(0, 0, 0, 0)
elif _api.USED_API == _api.QT_API_PYQT4:
vlayout.setMargin(0)
self.setLayout(vlayout)
self._scroll = QScrollBar(QtCore.Qt.Horizontal)
scrollarea = QScrollArea()
scrollarea.setMinimumHeight(140)
scrollarea.setWidgetResizable(True)
self._events_widget = EventsWidget(scroll=self._scroll)
scrollarea.setWidget(self._events_widget)
self._scroll.actionTriggered.connect(self.__scroll_changed)
vlayout.addWidget(scrollarea) # The timeline widget
vlayout.addWidget(self._scroll) # Add scroll
self._scroll.setMaximum(0)
self._scroll.setSliderPosition(0)
class OWPreprocess(OWWidget):
name = 'Preprocess Text'
description = 'Construct a text pre-processing pipeline.'
icon = 'icons/TextPreprocess.svg'
priority = 30
class Inputs:
corpus = Input("Corpus", Corpus)
class Outputs:
corpus = Output("Corpus", Corpus)
autocommit = settings.Setting(True)
preprocessors = [
TransformationModule,
TokenizerModule,
NormalizationModule,
FilteringModule,
NgramsModule,
POSTaggingModule,
]
transformers = settings.SettingProvider(TransformationModule)
tokenizer = settings.SettingProvider(TokenizerModule)
normalizer = settings.SettingProvider(NormalizationModule)
filters = settings.SettingProvider(FilteringModule)
ngrams_range = settings.SettingProvider(NgramsModule)
pos_tagger = settings.SettingProvider(POSTaggingModule)
control_area_width = 250
buttons_area_orientation = Qt.Vertical
UserAdviceMessages = [
widget.Message(
"Some preprocessing methods require data (like word relationships, stop words, "
"punctuation rules etc.) from the NLTK package. This data was downloaded "
"to: {}".format(nltk_data_dir()), "nltk_data")
]
class Error(OWWidget.Error):
stanford_tagger = Msg("Problem while loading Stanford POS Tagger\n{}")
class Warning(OWWidget.Warning):
no_token_left = Msg(
'No tokens on output! Please, change configuration.')
def __init__(self, parent=None):
super().__init__(parent)
self.corpus = None
self.initial_ngram_range = None # initial range of input corpus — used for inplace
self.preprocessor = preprocess.Preprocessor()
# -- INFO --
info_box = gui.widgetBox(self.controlArea, 'Info')
info_box.setFixedWidth(self.control_area_width)
self.controlArea.layout().addStretch()
self.info_label = gui.label(info_box, self, '')
self.update_info()
# -- PIPELINE --
frame = QFrame()
frame.setContentsMargins(0, 0, 0, 0)
frame.setFrameStyle(QFrame.Box)
frame.setStyleSheet('.QFrame { border: 1px solid #B3B3B3; }')
frame_layout = QVBoxLayout()
frame_layout.setContentsMargins(0, 0, 0, 0)
frame_layout.setSpacing(0)
frame.setLayout(frame_layout)
self.stages = []
for stage in self.preprocessors:
widget = stage(self)
self.stages.append(widget)
setattr(self, stage.attribute, widget)
frame_layout.addWidget(widget)
widget.change_signal.connect(self.settings_invalidated)
frame_layout.addStretch()
self.scroll = QScrollArea()
self.scroll.setWidget(frame)
self.scroll.setWidgetResizable(True)
self.scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
self.scroll.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
self.scroll.resize(frame_layout.sizeHint())
self.scroll.setMinimumHeight(500)
self.set_minimal_width()
self.mainArea.layout().addWidget(self.scroll)
# Buttons area
self.report_button.setFixedWidth(self.control_area_width)
commit_button = gui.auto_commit(self.buttonsArea,
self,
'autocommit',
'Commit',
box=False)
commit_button.setFixedWidth(self.control_area_width - 5)
self.buttonsArea.layout().addWidget(commit_button)
@Inputs.corpus
def set_data(self, data=None):
self.corpus = data.copy() if data is not None else None
self.initial_ngram_range = data.ngram_range if data is not None else None
self.commit()
def update_info(self, corpus=None):
if corpus is not None:
info = 'Document count: {}\n' \
'Total tokens: {}\n'\
'Total types: {}'\
.format(len(corpus), sum(map(len, corpus.tokens)), len(corpus.dictionary))
else:
info = 'No corpus.'
self.info_label.setText(info)
def commit(self):
self.Warning.no_token_left.clear()
if self.corpus is not None:
self.apply()
else:
self.update_info()
self.Outputs.corpus.send(None)
def apply(self):
self.preprocess()
@asynchronous
def preprocess(self):
for module in self.stages:
setattr(self.preprocessor, module.attribute, module.value)
self.corpus.pos_tags = None # reset pos_tags and ngrams_range
self.corpus.ngram_range = self.initial_ngram_range
return self.preprocessor(self.corpus,
inplace=True,
on_progress=self.on_progress)
@preprocess.on_start
def on_start(self):
self.progressBarInit(None)
@preprocess.callback
def on_progress(self, i):
self.progressBarSet(i, None)
@preprocess.on_result
def on_result(self, result):
self.update_info(result)
if result is not None and len(result.dictionary) == 0:
self.Warning.no_token_left()
result = None
self.Outputs.corpus.send(result)
self.progressBarFinished(None)
def set_minimal_width(self):
max_width = 250
for widget in self.stages:
if widget.enabled:
max_width = max(max_width, widget.sizeHint().width())
self.scroll.setMinimumWidth(max_width + 20)
@pyqtSlot()
def settings_invalidated(self):
self.set_minimal_width()
self.commit()
def send_report(self):
self.report_items('Preprocessor', self.preprocessor.report())
class CollapsibleFilterComponent(OWComponent, QObject):
options_changed = pyqtSignal()
filter_by_full_text: str = settings.Setting('', schema_only=True)
filter_by_name: str = settings.Setting('', schema_only=True)
filter_by_contrib: str = settings.Setting('', schema_only=True)
filter_by_owner: str = settings.Setting('', schema_only=True)
filter_by_modified: int = settings.Setting(FilterByDateModified.any_time, schema_only=True)
sort_by: int = settings.Setting(SortBy.newest_first, schema_only=True)
FILTER_FULL_TEXT_LABEL = 'Search'
TOGGLE_BTN_LABEL = 'Narrow your search'
FILTER_NAME_LABEL = 'Filter by name'
FILTER_CONTRIB_LABEL = 'Filter by contributor'
FILTER_OWNER_LABEL = 'Filter by owner'
FILTER_MODIFIED_LABEL = 'Filter by date modified'
SORTING_LABEL = 'Sorting'
def __init__(self, parent_widget, parent_component):
QObject.__init__(self)
OWComponent.__init__(self, widget=parent_widget)
box = gui.widgetBox(parent_component, margin=0)
self.filter_full_text = gui.lineEdit(
box,
self,
'filter_by_full_text',
label=self.FILTER_FULL_TEXT_LABEL,
callback=self.on_filter_full_text_changed,
)
self.toggle_animation = QParallelAnimationGroup()
self.toggle_button = QToolButton()
self.toggle_button.setCheckable(True)
self.toggle_button.setChecked(False)
self.toggle_button.setToolButtonStyle(Qt.ToolButtonTextBesideIcon)
self.toggle_button.setArrowType(Qt.RightArrow)
self.toggle_button.setText(self.TOGGLE_BTN_LABEL)
self.toggle_button.setStyleSheet('QToolButton {border: none; padding-top: 5px; }')
self.toggle_button.setIconSize(QSize(15, 15))
self.toggle_button.pressed.connect(self.on_toggle)
self.collapsible_components = QScrollArea()
self.collapsible_components.setMaximumHeight(0)
self.collapsible_components.setMinimumHeight(0)
self.collapsible_components.setSizePolicy(QSizePolicy.Expanding, QSizePolicy.Fixed)
self.collapsible_components.setFrameShape(QFrame.NoFrame)
box = gui.widgetBox(parent_component, margin=0)
box.layout().addWidget(self.toggle_button)
box.layout().addWidget(self.collapsible_components)
self.toggle_animation.addAnimation(QPropertyAnimation(box, b"minimumHeight"))
self.toggle_animation.addAnimation(QPropertyAnimation(box, b"maximumHeight"))
self.toggle_animation.addAnimation(QPropertyAnimation(self.collapsible_components, b"maximumHeight"))
layout = QHBoxLayout()
left_box = gui.widgetBox(None, self, margin=0, flat=True)
mid_box = gui.widgetBox(None, self, margin=0, flat=True)
right_box = gui.widgetBox(None, self, margin=0, flat=True)
self.filter_name = gui.lineEdit(
left_box, self, 'filter_by_name', label=self.FILTER_NAME_LABEL, callback=self.on_filter_changed, addSpace=5
)
self.filter_contrib = gui.lineEdit(
mid_box,
self,
'filter_by_contrib',
label=self.FILTER_CONTRIB_LABEL,
callback=self.on_filter_changed,
addSpace=5,
)
self.filter_owner = gui.lineEdit(
right_box,
self,
'filter_by_owner',
label=self.FILTER_OWNER_LABEL,
callback=self.on_filter_changed,
addSpace=5,
)
self.filter_modified = gui.comboBox(
left_box,
self,
'filter_by_modified',
label=self.FILTER_MODIFIED_LABEL,
callback=self.on_filter_changed,
items=FilterByDateModified.labels(),
)
self.sorting = gui.comboBox(
mid_box, self, 'sort_by', label=self.SORTING_LABEL, callback=self.on_filter_changed, items=SortBy.labels()
)
gui.rubber(left_box)
gui.rubber(mid_box)
gui.rubber(right_box)
layout.addWidget(left_box)
layout.addWidget(mid_box)
layout.addWidget(right_box)
self.collapsible_components.setLayout(layout)
collapsed_height = box.layout().sizeHint().height() - self.collapsible_components.maximumHeight()
content_height = layout.sizeHint().height()
for i in range(self.toggle_animation.animationCount()):
animation = self.toggle_animation.animationAt(i)
animation.setDuration(100)
animation.setStartValue(collapsed_height)
animation.setEndValue(collapsed_height + content_height)
content_animation = self.toggle_animation.animationAt(self.toggle_animation.animationCount() - 1)
content_animation.setDuration(100)
content_animation.setStartValue(0)
content_animation.setEndValue(content_height)
def on_toggle(self):
""" Start animation """
checked = self.toggle_button.isChecked()
self.toggle_button.setArrowType(Qt.DownArrow if not checked else Qt.RightArrow)
self.toggle_animation.setDirection(QAbstractAnimation.Forward if not checked else QAbstractAnimation.Backward)
self.toggle_animation.start()
def on_filter_full_text_changed(self):
self.sort_by = SortBy.relevance if self.filter_full_text else SortBy.newest_first
self.on_filter_changed()
def on_filter_changed(self):
self.options_changed.emit()
class OWPreprocess(OWWidget):
name = 'Preprocess Text'
description = 'Construct a text pre-processing pipeline.'
icon = 'icons/TextPreprocess.svg'
priority = 30
inputs = [(Input.CORPUS, Corpus, 'set_data')]
outputs = [(Output.PP_CORPUS, Corpus)]
autocommit = settings.Setting(True)
preprocessors = [
TransformationModule,
TokenizerModule,
NormalizationModule,
FilteringModule,
NgramsModule,
POSTaggingModule,
]
transformers = settings.SettingProvider(TransformationModule)
tokenizer = settings.SettingProvider(TokenizerModule)
normalizer = settings.SettingProvider(NormalizationModule)
filters = settings.SettingProvider(FilteringModule)
ngrams_range = settings.SettingProvider(NgramsModule)
pos_tagger = settings.SettingProvider(POSTaggingModule)
control_area_width = 250
buttons_area_orientation = Qt.Vertical
UserAdviceMessages = [
widget.Message(
"Some preprocessing methods require data (like word relationships, stop words, "
"punctuation rules etc.) from the NLTK package. This data, if you didn't have it "
"already, was downloaded to: {}".format(Downloader().default_download_dir()),
"nltk_data")]
class Error(OWWidget.Error):
stanford_tagger = Msg("Problem while loading Stanford POS Tagger\n{}")
class Warning(OWWidget.Warning):
no_token_left = Msg('No tokens on output! Please, change configuration.')
def __init__(self, parent=None):
super().__init__(parent)
self.corpus = None
self.initial_ngram_range = None # initial range of input corpus — used for inplace
self.preprocessor = preprocess.Preprocessor()
# -- INFO --
info_box = gui.widgetBox(self.controlArea, 'Info')
info_box.setFixedWidth(self.control_area_width)
self.controlArea.layout().addStretch()
self.info_label = gui.label(info_box, self, '')
self.update_info()
# -- PIPELINE --
frame = QFrame()
frame.setContentsMargins(0, 0, 0, 0)
frame.setFrameStyle(QFrame.Box)
frame.setStyleSheet('.QFrame { border: 1px solid #B3B3B3; }')
frame_layout = QVBoxLayout()
frame_layout.setContentsMargins(0, 0, 0, 0)
frame_layout.setSpacing(0)
frame.setLayout(frame_layout)
self.stages = []
for stage in self.preprocessors:
widget = stage(self)
self.stages.append(widget)
setattr(self, stage.attribute, widget)
frame_layout.addWidget(widget)
widget.change_signal.connect(self.settings_invalidated)
frame_layout.addStretch()
self.scroll = QScrollArea()
self.scroll.setWidget(frame)
self.scroll.setWidgetResizable(True)
self.scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
self.scroll.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
self.scroll.resize(frame_layout.sizeHint())
self.scroll.setMinimumHeight(500)
self.set_minimal_width()
self.mainArea.layout().addWidget(self.scroll)
# Buttons area
self.report_button.setFixedWidth(self.control_area_width)
commit_button = gui.auto_commit(self.buttonsArea, self, 'autocommit',
'Commit', box=False)
commit_button.setFixedWidth(self.control_area_width - 5)
self.buttonsArea.layout().addWidget(commit_button)
def set_data(self, data=None):
self.corpus = data.copy() if data is not None else None
self.initial_ngram_range = data.ngram_range if data is not None else None
self.commit()
def update_info(self, corpus=None):
if corpus is not None:
info = 'Document count: {}\n' \
'Total tokens: {}\n'\
'Total types: {}'\
.format(len(corpus), sum(map(len, corpus.tokens)), len(corpus.dictionary))
else:
info = 'No corpus.'
self.info_label.setText(info)
def commit(self):
self.Warning.no_token_left.clear()
if self.corpus is not None:
self.apply()
else:
self.update_info()
self.send(Output.PP_CORPUS, None)
def apply(self):
self.preprocess()
@asynchronous
def preprocess(self):
for module in self.stages:
setattr(self.preprocessor, module.attribute, module.value)
self.corpus.pos_tags = None # reset pos_tags and ngrams_range
self.corpus.ngram_range = self.initial_ngram_range
return self.preprocessor(self.corpus, inplace=True, on_progress=self.on_progress)
@preprocess.on_start
def on_start(self):
self.progressBarInit(None)
@preprocess.callback
def on_progress(self, i):
self.progressBarSet(i, None)
@preprocess.on_result
def on_result(self, result):
self.update_info(result)
if result is not None and len(result.dictionary) == 0:
self.Warning.no_token_left()
result = None
self.send(Output.PP_CORPUS, result)
self.progressBarFinished(None)
def set_minimal_width(self):
max_width = 250
for widget in self.stages:
if widget.enabled:
max_width = max(max_width, widget.sizeHint().width())
self.scroll.setMinimumWidth(max_width + 20)
@pyqtSlot()
def settings_invalidated(self):
self.set_minimal_width()
self.commit()
def send_report(self):
self.report_items('Preprocessor', self.preprocessor.report())
class OWPreprocess(OWWidget):
name = '文本预处理'
description = '构建文本预处理的管道'
icon = 'icons/TextPreprocess.svg'
priority = 200
class Inputs:
corpus = Input("Corpus", Corpus)
class Outputs:
corpus = Output("Corpus", Corpus)
autocommit = settings.Setting(True)
preprocessors = [
TransformationModule,
TokenizerModule,
NormalizationModule,
FilteringModule,
NgramsModule,
POSTaggingModule,
]
transformers = settings.SettingProvider(TransformationModule)
tokenizer = settings.SettingProvider(TokenizerModule)
normalizer = settings.SettingProvider(NormalizationModule)
filters = settings.SettingProvider(FilteringModule)
ngrams_range = settings.SettingProvider(NgramsModule)
pos_tagger = settings.SettingProvider(POSTaggingModule)
control_area_width = 250
buttons_area_orientation = Qt.Vertical
UserAdviceMessages = [
widget.Message("部分预处理所需要的数据（例如词汇关系、停用词、标点符号规则等）是从NLTK包中获取的，",
"这些数据可以从{}下载。".format(nltk_data_dir()))
]
class Error(OWWidget.Error):
stanford_tagger = Msg("无法加载Stanford POS Tagger\n{}")
stopwords_encoding = Msg("停用词表编码不正确，请使用 UTF-8 再试一次。")
lexicon_encoding = Msg("词典编码不正确，请使用 UTF-8 再试一次。")
error_reading_stopwords = Msg("读取文件错误: {}")
error_reading_lexicon = Msg("读取文件错误: {}")
class Warning(OWWidget.Warning):
no_token_left = Msg('没有标记输出，请重新配置')
udpipe_offline = Msg('没有网络连接，UDPipe 只加载本地模型')
udpipe_offline_no_models = Msg('没有网络连接，UDPipe无本地模型')
def __init__(self, parent=None):
super().__init__(parent)
self.corpus = None
self.initial_ngram_range = None # initial range of input corpus — used for inplace
self.preprocessor = preprocess.Preprocessor()
# -- INFO --
info_box = gui.widgetBox(self.controlArea, '基本信息')
info_box.setFixedWidth(self.control_area_width)
self.controlArea.layout().addStretch()
self.info_label = gui.label(info_box, self, '')
self.update_info()
# -- PIPELINE --
frame = QFrame()
frame.setContentsMargins(0, 0, 0, 0)
frame.setFrameStyle(QFrame.Box)
frame.setStyleSheet('.QFrame { border: 1px solid #B3B3B3; }')
frame_layout = QVBoxLayout()
frame_layout.setContentsMargins(0, 0, 0, 0)
frame_layout.setSpacing(0)
frame.setLayout(frame_layout)
self.stages = []
for stage in self.preprocessors:
widget = stage(self)
self.stages.append(widget)
setattr(self, stage.attribute, widget)
frame_layout.addWidget(widget)
widget.change_signal.connect(self.settings_invalidated)
frame_layout.addStretch()
self.scroll = QScrollArea()
self.scroll.setWidget(frame)
self.scroll.setWidgetResizable(True)
self.scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff)
self.scroll.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
self.scroll.resize(frame_layout.sizeHint())
self.scroll.setMinimumHeight(500)
self.set_minimal_width()
self.mainArea.layout().addWidget(self.scroll)
# Buttons area
self.report_button.setFixedWidth(self.control_area_width)
commit_button = gui.auto_commit(self.buttonsArea,
self,
'autocommit',
'提交',
'自动提交',
box=False)
commit_button.setFixedWidth(self.control_area_width - 5)
self.buttonsArea.layout().addWidget(commit_button)
@Inputs.corpus
def set_data(self, data=None):
self.corpus = data.copy() if data is not None else None
self.initial_ngram_range = data.ngram_range if data is not None else None
self.commit()
def update_info(self, corpus=None):
if corpus is not None:
info = '文档数量: {}\n' \
'标记数量: {}\n'\
'类型数量: {}'\
.format(len(corpus), sum(map(len, corpus.tokens)), len(corpus.dictionary))
else:
info = '没有数据集'
self.info_label.setText(info)
def commit(self):
self.Warning.no_token_left.clear()
if self.corpus is not None:
self.apply()
else:
self.update_info()
self.Outputs.corpus.send(None)
def apply(self):
self.preprocess()
@asynchronous
def preprocess(self):
for module in self.stages:
setattr(self.preprocessor, module.attribute, module.value)
self.corpus.pos_tags = None # reset pos_tags and ngrams_range
self.corpus.ngram_range = self.initial_ngram_range
return self.preprocessor(self.corpus,
inplace=True,
on_progress=self.on_progress)
@preprocess.on_start
def on_start(self):
self.progressBarInit(None)
@preprocess.callback
def on_progress(self, i):
self.progressBarSet(i, None)
@preprocess.on_result
def on_result(self, result):
self.update_info(result)
if result is not None and len(result.dictionary) == 0:
self.Warning.no_token_left()
result = None
self.Outputs.corpus.send(result)
self.progressBarFinished(None)
def set_minimal_width(self):
max_width = 250
for widget in self.stages:
if widget.enabled:
max_width = max(max_width, widget.sizeHint().width())
self.scroll.setMinimumWidth(max_width + 20)
@pyqtSlot()
def settings_invalidated(self):
self.set_minimal_width()
self.commit()
def send_report(self):
self.report_items('Preprocessor', self.preprocessor.report())
def init_form(self):
# Get the current path of the file
rootPath = os.path.dirname(__file__)
vlayout = QVBoxLayout()
hlayout = QHBoxLayout()
if _api.USED_API == _api.QT_API_PYQT5:
hlayout.setContentsMargins(0, 0, 0, 0)
vlayout.setContentsMargins(0, 0, 0, 0)
elif _api.USED_API == _api.QT_API_PYQT4:
hlayout.setMargin(0)
vlayout.setMargin(0)
self.setLayout(vlayout)
# Add scroll area
scrollarea = QScrollArea()
self._scrollArea = scrollarea
scrollarea.setMinimumHeight(140)
scrollarea.setWidgetResizable(True)
scrollarea.keyPressEvent = self.__scrollAreaKeyPressEvent
scrollarea.keyReleaseEvent = self.__scrollAreaKeyReleaseEvent
vlayout.addWidget(scrollarea)
# The timeline widget
self._time = widget = TimelineWidget(self)
widget._scroll = scrollarea
scrollarea.setWidget(widget)
# Timeline zoom slider
slider = QSlider(QtCore.Qt.Horizontal)
slider.setFocusPolicy(QtCore.Qt.NoFocus)
slider.setMinimum(1)
slider.setMaximum(100)
slider.setValue(10)
slider.setPageStep(1)
slider.setTickPosition(QSlider.NoTicks) # TicksBothSides
slider.valueChanged.connect(self.__scaleSliderChange)
slider_label_zoom_in = QLabel()
slider_label_zoom_out = QLabel()
slider_label_zoom_in.setPixmap(
conf.PYFORMS_PIXMAP_EVENTTIMELINE_ZOOM_IN)
slider_label_zoom_out.setPixmap(
conf.PYFORMS_PIXMAP_EVENTTIMELINE_ZOOM_OUT)
self._zoomLabel = QLabel("100%")
hlayout.addWidget(self._zoomLabel)
hlayout.addWidget(slider_label_zoom_out)
hlayout.addWidget(slider)
hlayout.addWidget(slider_label_zoom_in)
# Import/Export Buttons
btn_import = QPushButton("Import")
btn_import.setIcon(conf.PYFORMS_ICON_EVENTTIMELINE_IMPORT)
btn_import.clicked.connect(self.__open_import_win_evt)
btn_export = QPushButton("Export")
btn_export.setIcon(conf.PYFORMS_ICON_EVENTTIMELINE_EXPORT)
btn_export.clicked.connect(self.__export)
hlayout.addWidget(btn_import)
hlayout.addWidget(btn_export)
vlayout.addLayout(hlayout)
