>>> new_corpus.tokens[0][:10] ['hous', 'say', ';', 'spoke', 'littl', 'one', 'hand', 'wall', 'hurt', '?'] This module defines ``default_preprocessor`` that will be used to extract tokens from a :ref:`Corpus` if no preprocessing was applied yet:: >>> from orangecontrib.text import Corpus >>> corpus = Corpus.from_file('deerwester') >>> corpus.tokens[0] ['human', 'machine', 'interface', 'for', 'lab', 'abc', 'computer', 'applications'] """ import nltk from orangecontrib.text.misc import nltk_data_dir # Ensure NLTK data is present NLTK_DATA = [ "wordnet", "stopwords", "punkt", "opinion_lexicon", "vader_lexicon" ] nltk.download(NLTK_DATA, download_dir=nltk_data_dir(), quiet=True) from .filter import * from .normalize import * from .tokenize import * from .transform import * from .preprocess import * base_preprocessor = Preprocessor(transformers=[LowercaseTransformer()], tokenizer=WordPunctTokenizer())
they not set. """ def add_scheme(url: Optional[str]) -> Optional[str]: if url is not None and "://" not in url: # if no scheme default to http - as other libraries do (e.g. requests) return f"http://{url}" else: return url http_proxy = add_scheme(os.environ.get("http_proxy")) https_proxy = add_scheme(os.environ.get("https_proxy")) proxy_dict = {} if http_proxy: proxy_dict["http://"] = http_proxy if https_proxy: proxy_dict["https://"] = https_proxy return proxy_dict if proxy_dict else None embedder_utils.get_proxies = _get_proxies # remove to here from orangecontrib.text.misc import nltk_data_dir os.environ['NLTK_DATA'] = nltk_data_dir() from .corpus import Corpus from .version import git_revision as __git_revision__ from .version import version as __version__
class OWPreprocess(OWWidget): name = 'Preprocess Text' description = 'Construct a text pre-processing pipeline.' icon = 'icons/TextPreprocess.svg' priority = 30 class Inputs: corpus = Input("Corpus", Corpus) class Outputs: corpus = Output("Corpus", Corpus) autocommit = settings.Setting(True) preprocessors = [ TransformationModule, TokenizerModule, NormalizationModule, FilteringModule, NgramsModule, POSTaggingModule, ] transformers = settings.SettingProvider(TransformationModule) tokenizer = settings.SettingProvider(TokenizerModule) normalizer = settings.SettingProvider(NormalizationModule) filters = settings.SettingProvider(FilteringModule) ngrams_range = settings.SettingProvider(NgramsModule) pos_tagger = settings.SettingProvider(POSTaggingModule) control_area_width = 250 buttons_area_orientation = Qt.Vertical UserAdviceMessages = [ widget.Message( "Some preprocessing methods require data (like word relationships, stop words, " "punctuation rules etc.) from the NLTK package. This data was downloaded " "to: {}".format(nltk_data_dir()), "nltk_data") ] class Error(OWWidget.Error): stanford_tagger = Msg("Problem while loading Stanford POS Tagger\n{}") class Warning(OWWidget.Warning): no_token_left = Msg( 'No tokens on output! Please, change configuration.') def __init__(self, parent=None): super().__init__(parent) self.corpus = None self.initial_ngram_range = None # initial range of input corpus — used for inplace self.preprocessor = preprocess.Preprocessor() # -- INFO -- info_box = gui.widgetBox(self.controlArea, 'Info') info_box.setFixedWidth(self.control_area_width) self.controlArea.layout().addStretch() self.info_label = gui.label(info_box, self, '') self.update_info() # -- PIPELINE -- frame = QFrame() frame.setContentsMargins(0, 0, 0, 0) frame.setFrameStyle(QFrame.Box) frame.setStyleSheet('.QFrame { border: 1px solid #B3B3B3; }') frame_layout = QVBoxLayout() frame_layout.setContentsMargins(0, 0, 0, 0) frame_layout.setSpacing(0) frame.setLayout(frame_layout) self.stages = [] for stage in self.preprocessors: widget = stage(self) self.stages.append(widget) setattr(self, stage.attribute, widget) frame_layout.addWidget(widget) widget.change_signal.connect(self.settings_invalidated) frame_layout.addStretch() self.scroll = QScrollArea() self.scroll.setWidget(frame) self.scroll.setWidgetResizable(True) self.scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) self.scroll.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.scroll.resize(frame_layout.sizeHint()) self.scroll.setMinimumHeight(500) self.set_minimal_width() self.mainArea.layout().addWidget(self.scroll) # Buttons area self.report_button.setFixedWidth(self.control_area_width) commit_button = gui.auto_commit(self.buttonsArea, self, 'autocommit', 'Commit', box=False) commit_button.setFixedWidth(self.control_area_width - 5) self.buttonsArea.layout().addWidget(commit_button) @Inputs.corpus def set_data(self, data=None): self.corpus = data.copy() if data is not None else None self.initial_ngram_range = data.ngram_range if data is not None else None self.commit() def update_info(self, corpus=None): if corpus is not None: info = 'Document count: {}\n' \ 'Total tokens: {}\n'\ 'Total types: {}'\ .format(len(corpus), sum(map(len, corpus.tokens)), len(corpus.dictionary)) else: info = 'No corpus.' self.info_label.setText(info) def commit(self): self.Warning.no_token_left.clear() if self.corpus is not None: self.apply() else: self.update_info() self.Outputs.corpus.send(None) def apply(self): self.preprocess() @asynchronous def preprocess(self): for module in self.stages: setattr(self.preprocessor, module.attribute, module.value) self.corpus.pos_tags = None # reset pos_tags and ngrams_range self.corpus.ngram_range = self.initial_ngram_range return self.preprocessor(self.corpus, inplace=True, on_progress=self.on_progress) @preprocess.on_start def on_start(self): self.progressBarInit(None) @preprocess.callback def on_progress(self, i): self.progressBarSet(i, None) @preprocess.on_result def on_result(self, result): self.update_info(result) if result is not None and len(result.dictionary) == 0: self.Warning.no_token_left() result = None self.Outputs.corpus.send(result) self.progressBarFinished(None) def set_minimal_width(self): max_width = 250 for widget in self.stages: if widget.enabled: max_width = max(max_width, widget.sizeHint().width()) self.scroll.setMinimumWidth(max_width + 20) @pyqtSlot() def settings_invalidated(self): self.set_minimal_width() self.commit() def send_report(self): self.report_items('Preprocessor', self.preprocessor.report())
class OWPreprocess(OWWidget): name = '文本预处理' description = '构建文本预处理的管道' icon = 'icons/TextPreprocess.svg' priority = 200 class Inputs: corpus = Input("Corpus", Corpus) class Outputs: corpus = Output("Corpus", Corpus) autocommit = settings.Setting(True) preprocessors = [ TransformationModule, TokenizerModule, NormalizationModule, FilteringModule, NgramsModule, POSTaggingModule, ] transformers = settings.SettingProvider(TransformationModule) tokenizer = settings.SettingProvider(TokenizerModule) normalizer = settings.SettingProvider(NormalizationModule) filters = settings.SettingProvider(FilteringModule) ngrams_range = settings.SettingProvider(NgramsModule) pos_tagger = settings.SettingProvider(POSTaggingModule) control_area_width = 250 buttons_area_orientation = Qt.Vertical UserAdviceMessages = [ widget.Message("部分预处理所需要的数据(例如词汇关系、停用词、标点符号规则等)是从NLTK包中获取的,", "这些数据可以从{}下载。".format(nltk_data_dir())) ] class Error(OWWidget.Error): stanford_tagger = Msg("无法加载Stanford POS Tagger\n{}") stopwords_encoding = Msg("停用词表编码不正确,请使用 UTF-8 再试一次。") lexicon_encoding = Msg("词典编码不正确,请使用 UTF-8 再试一次。") error_reading_stopwords = Msg("读取文件错误: {}") error_reading_lexicon = Msg("读取文件错误: {}") class Warning(OWWidget.Warning): no_token_left = Msg('没有标记输出,请重新配置') udpipe_offline = Msg('没有网络连接,UDPipe 只加载本地模型') udpipe_offline_no_models = Msg('没有网络连接,UDPipe无本地模型') def __init__(self, parent=None): super().__init__(parent) self.corpus = None self.initial_ngram_range = None # initial range of input corpus — used for inplace self.preprocessor = preprocess.Preprocessor() # -- INFO -- info_box = gui.widgetBox(self.controlArea, '基本信息') info_box.setFixedWidth(self.control_area_width) self.controlArea.layout().addStretch() self.info_label = gui.label(info_box, self, '') self.update_info() # -- PIPELINE -- frame = QFrame() frame.setContentsMargins(0, 0, 0, 0) frame.setFrameStyle(QFrame.Box) frame.setStyleSheet('.QFrame { border: 1px solid #B3B3B3; }') frame_layout = QVBoxLayout() frame_layout.setContentsMargins(0, 0, 0, 0) frame_layout.setSpacing(0) frame.setLayout(frame_layout) self.stages = [] for stage in self.preprocessors: widget = stage(self) self.stages.append(widget) setattr(self, stage.attribute, widget) frame_layout.addWidget(widget) widget.change_signal.connect(self.settings_invalidated) frame_layout.addStretch() self.scroll = QScrollArea() self.scroll.setWidget(frame) self.scroll.setWidgetResizable(True) self.scroll.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOff) self.scroll.setVerticalScrollBarPolicy(Qt.ScrollBarAlwaysOn) self.scroll.resize(frame_layout.sizeHint()) self.scroll.setMinimumHeight(500) self.set_minimal_width() self.mainArea.layout().addWidget(self.scroll) # Buttons area self.report_button.setFixedWidth(self.control_area_width) commit_button = gui.auto_commit(self.buttonsArea, self, 'autocommit', '提交', '自动提交', box=False) commit_button.setFixedWidth(self.control_area_width - 5) self.buttonsArea.layout().addWidget(commit_button) @Inputs.corpus def set_data(self, data=None): self.corpus = data.copy() if data is not None else None self.initial_ngram_range = data.ngram_range if data is not None else None self.commit() def update_info(self, corpus=None): if corpus is not None: info = '文档数量: {}\n' \ '标记数量: {}\n'\ '类型数量: {}'\ .format(len(corpus), sum(map(len, corpus.tokens)), len(corpus.dictionary)) else: info = '没有数据集' self.info_label.setText(info) def commit(self): self.Warning.no_token_left.clear() if self.corpus is not None: self.apply() else: self.update_info() self.Outputs.corpus.send(None) def apply(self): self.preprocess() @asynchronous def preprocess(self): for module in self.stages: setattr(self.preprocessor, module.attribute, module.value) self.corpus.pos_tags = None # reset pos_tags and ngrams_range self.corpus.ngram_range = self.initial_ngram_range return self.preprocessor(self.corpus, inplace=True, on_progress=self.on_progress) @preprocess.on_start def on_start(self): self.progressBarInit(None) @preprocess.callback def on_progress(self, i): self.progressBarSet(i, None) @preprocess.on_result def on_result(self, result): self.update_info(result) if result is not None and len(result.dictionary) == 0: self.Warning.no_token_left() result = None self.Outputs.corpus.send(result) self.progressBarFinished(None) def set_minimal_width(self): max_width = 250 for widget in self.stages: if widget.enabled: max_width = max(max_width, widget.sizeHint().width()) self.scroll.setMinimumWidth(max_width + 20) @pyqtSlot() def settings_invalidated(self): self.set_minimal_width() self.commit() def send_report(self): self.report_items('Preprocessor', self.preprocessor.report())