class SpaCy(OWTextableBaseWidget):
"""Textable widget for NLP using spaCy."""
#----------------------------------------------------------------------
# Widget's metadata...
name = "spaCy"
description = "Natural language processing using spaCy"
icon = "icons/spacy.svg"
priority = 21 # TODO
#----------------------------------------------------------------------
# Channel definitions...
inputs = [("Text data", Segmentation, "inputData")]
outputs = [("Linguistically analyzed data", Segmentation)]
#----------------------------------------------------------------------
# Layout parameters...
want_main_area = False
#----------------------------------------------------------------------
# Settings...
settingsHandler = VersionedSettingsHandler(
version=__version__.rsplit(".", 1)[0])
autoSend = settings.Setting(False)
model = settings.Setting("fr_core_news_sm")
def __init__(self):
"""Widget creator."""
super().__init__()
# Other attributes...
self.inputSeg = None
self.nlp = None
# Next two instructions are helpers from TextableUtils. Corresponding
# interface elements are declared here and actually drawn below (at
# their position in the UI)...
self.infoBox = InfoBox(widget=self.controlArea)
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute="infoBox",
sendIfPreCallback=None,
)
# User interface...
# Tabs...
self.tabs = QTabWidget()
self.optionsTab = QWidget()
self.modelManagerTab = QWidget()
self.tabs.addTab(self.optionsTab, "Options")
self.tabs.addTab(self.modelManagerTab, "Model manager")
# Options tab...
OptionsTabBox = QHBoxLayout()
optionsBox = gui.widgetBox(widget=self.optionsTab)
self.modelComboBox = gui.comboBox(
widget=optionsBox,
master=self,
value='model',
label='Model: ',
tooltip='Select the spaCy language model you want to use.',
items=INSTALLED_MODELS,
sendSelectedValue=True,
callback=self.modelChanged,
)
OptionsTabBox.addWidget(optionsBox)
self.optionsTab.setLayout(OptionsTabBox)
# Model manager tab...
modelManagerTabBox = QHBoxLayout()
modelManagerBox = gui.widgetBox(widget=self.modelManagerTab)
# TODO: Model manager UI
modelManagerTabBox.addWidget(modelManagerBox)
self.modelManagerTab.setLayout(modelManagerTabBox)
self.controlArea.layout().addWidget(self.tabs)
gui.rubber(self.controlArea)
# Now Info box and Send button must be drawn...
self.sendButton.draw()
self.infoBox.draw()
self.infoBox.setText("Widget needs input", "warning")
# Load spaCy language model...
self.modelChanged()
# Send data if autoSend.
self.sendButton.sendIf()
def inputData(self, newInput):
"""Process incoming data."""
self.inputSeg = newInput
self.infoBox.inputChanged()
self.sendButton.sendIf()
def modelChanged(self):
"""Respond to model change in UI."""
self.nlp = spacy.load(self.model)
self.sendButton.settingsChanged()
def sendData(self):
"""Compute result of widget processing and send to output"""
# Check that there's an input...
if self.inputSeg is None:
self.infoBox.setText("Widget needs input", "warning")
self.send("Linguistically analyzed data", None, self)
return
# Initialize progress bar.
self.infoBox.setText(
u"Processing, please wait...",
"warning",
)
self.controlArea.setDisabled(True)
progressBar = ProgressBar(self, iterations=len(self.inputSeg))
tokenizedSegments = list()
# Process each input segment...
for segment in self.inputSeg:
# Input segment attributes...
inputContent = segment.get_content()
inputAnnotations = segment.annotations
inputString = segment.str_index
inputStart = segment.start or 0
inputEnd = segment.end or len(inputContent)
# NLP analysis...
doc = self.nlp(inputContent)
# Process each token in input segment...
for token in doc:
tokenAnnotations = inputAnnotations.copy()
tokenAnnotations.update({
k: getattr(token, k)
for k in RELEVANT_KEYS if getattr(token, k) is not None
})
tokenStart = inputStart + token.idx
tokenizedSegments.append(
Segment(
str_index=inputString,
start=tokenStart,
end=tokenStart + len(token),
annotations=tokenAnnotations,
))
progressBar.advance()
outputSeg = Segmentation(tokenizedSegments, self.captionTitle)
# Set status to OK and report data size...
message = "%i segment@p sent to output." % len(outputSeg)
message = pluralize(message, len(outputSeg))
self.infoBox.setText(message)
print(outputSeg.to_string())
# Clear progress bar.
progressBar.finish()
self.controlArea.setDisabled(False)
# Send data to output...
self.send("Linguistically analyzed data", outputSeg, self)
self.sendButton.resetSettingsChangedFlag()
# The following method needs to be copied verbatim in
# every Textable widget that sends a segmentation...
def setCaption(self, title):
if 'captionTitle' in dir(self):
changed = title != self.captionTitle
super().setCaption(title)
if changed:
self.sendButton.settingsChanged()
else:
super().setCaption(title)
def __init__(self):
"""Widget creator."""
super().__init__()
if INSTALLED_MODELS:
self.model = INSTALLED_MODELS[0]
else:
self.model = ""
# Other attributes...
self.inputSeg = None
self.nlp = None
self.selectedModels = list()
self.downloadableModelLabels = list()
self.loadedComponents = list()
self.mustLoad = True
# Next two instructions are helpers from TextableUtils. Corresponding
# interface elements are declared here and actually drawn below (at
# their position in the UI)...
self.infoBox = InfoBox(widget=self.controlArea)
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute="infoBox",
sendIfPreCallback=None,
)
# User interface...
# Tabs...
self.tabs = QTabWidget()
self.optionsTab = QWidget()
self.modelManagerTab = QWidget()
self.tabs.addTab(self.optionsTab, "Options")
self.tabs.addTab(self.modelManagerTab, "Model manager")
# Options tab...
OptionsTabBox = QHBoxLayout()
optionsBox = gui.widgetBox(widget=self.optionsTab)
self.modelComboBox = gui.comboBox(
widget=optionsBox,
master=self,
value='model',
label='Model: ',
tooltip='Select the spaCy language model you want to use.',
items=INSTALLED_MODELS[:],
sendSelectedValue=True,
callback=self.modelComboboxChanged,
)
gui.separator(widget=optionsBox, height=3)
annotationsBox = gui.widgetBox(
widget=optionsBox,
box="Additional token annotations:",
)
annotationsBoxLine1 = gui.widgetBox(
widget=annotationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=annotationsBoxLine1,
master=self,
value='annotatePOSTags',
label='part-of-speech tags',
callback=self.updateDisabledComponents,
tooltip=("Annotate output tokens with part-of-speech tags."),
)
self.annotatePOSTagsReloadLabel = gui.label(
annotationsBoxLine1,
master=self,
label="(reload needed)",
)
self.annotatePOSTagsReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
annotationsBoxLine2 = gui.widgetBox(
widget=annotationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=annotationsBoxLine2,
master=self,
value='annotateDependencies',
label='syntactic dependencies',
callback=self.updateDisabledComponents,
tooltip=("Annotate output tokens with syntactic dependencies."),
)
self.annotateDependenciesReloadLabel = gui.label(
annotationsBoxLine2,
master=self,
label="(reload needed)",
)
self.annotateDependenciesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
annotationsBoxLine3 = gui.widgetBox(
widget=annotationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=annotationsBoxLine3,
master=self,
value='annotateEntities',
label='named entities',
callback=self.updateDisabledComponents,
tooltip=("Annotate output tokens with named entities."),
)
self.annotateEntitiesReloadLabel = gui.label(
annotationsBoxLine3,
master=self,
label="(reload needed)",
)
self.annotateEntitiesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
segmentationsBox = gui.widgetBox(
widget=optionsBox,
box="Additional segmentations:",
)
segmentationsBoxLine1 = gui.widgetBox(
widget=segmentationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=segmentationsBoxLine1,
master=self,
value='segmentEntities',
label='named entities',
callback=self.updateDisabledComponents,
tooltip="Output named entity segmentation on separate channel.",
)
self.segmentEntitiesReloadLabel = gui.label(
segmentationsBoxLine1,
master=self,
label="(reload needed)",
)
self.segmentEntitiesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
segmentationsBoxLine2 = gui.widgetBox(
widget=segmentationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=segmentationsBoxLine2,
master=self,
value='segmentChunks',
label='noun chunks',
callback=self.updateDisabledComponents,
tooltip="Output noun chunk segmentation on separate channel.",
)
self.segmentChunksReloadLabel = gui.label(
segmentationsBoxLine2,
master=self,
label="(reload needed)",
)
self.segmentChunksReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
segmentationsBoxLine3 = gui.widgetBox(
widget=segmentationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=segmentationsBoxLine3,
master=self,
value='segmentSentences',
label='sentences',
callback=self.updateDisabledComponents,
tooltip="Output sentence segmentation on separate channel.",
)
self.segmentSentencesReloadLabel = gui.label(
segmentationsBoxLine3,
master=self,
label="(reload needed)",
)
self.segmentSentencesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
self.updateReloadNeededLabels()
gui.comboBox(
widget=optionsBox,
master=self,
value='maxLen',
items=["1 million"] + ["%i millions" % l for l in range(2, 10)] \
+ ["no limit"],
sendSelectedValue=True,
label=u'Max number of input characters:',
tooltip=(
"The spaCy parser and NER models require roughly 1GB of\n"
"temporary memory per 100'000 characters in the input.\n"
"This means long texts may cause memory allocation errors.\n"
"If you're not using the parser or NER, or have lots of \n"
"RAM, it's probably safe to increase the default limit of\n"
"1 million characters."
),
)
gui.rubber(optionsBox)
OptionsTabBox.addWidget(optionsBox)
self.optionsTab.setLayout(OptionsTabBox)
# Model manager tab...
modelManagerTabBox = QHBoxLayout()
modelManagerBox = gui.widgetBox(widget=self.modelManagerTab)
gui.label(modelManagerBox, self, label="Available models:")
self.downloadableModelsListbox = gui.listBox(
widget=modelManagerBox,
master=self,
value="selectedModels",
labels="downloadableModelLabels",
callback=self.downloadableModelsListboxChanged,
tooltip="Select language models then click Download.",
)
self.downloadableModelsListbox.setSelectionMode(3)
self.downloadableModelLabels = DOWNLOADABLE_MODELS[:]
self.downloadableModelLabels = self.downloadableModelLabels
self.downloadButton = gui.button(
widget=modelManagerBox,
master=self,
label="Download",
callback=self.downloadModels,
tooltip="Download the selected language models.",
)
self.downloadButton.setDisabled(True)
modelManagerTabBox.addWidget(modelManagerBox)
self.modelManagerTab.setLayout(modelManagerTabBox)
self.controlArea.layout().addWidget(self.tabs)
gui.rubber(self.controlArea)
# Now Info box and Send button must be drawn...
self.sendButton.draw()
self.infoBox.draw()
self.infoBox.setText("Widget needs input.", "warning")
# Check that there's a model...
if not self.model:
self.infoBox.setText(
"Please download a language model first.",
"warning",
)
self.tabs.setCurrentIndex(1)
optionsBox.setDisabled(True)
def __init__(self):
"""Widget creator."""
super().__init__()
# Other attributes...
self.inputSeg = None
self.nlp = None
# Next two instructions are helpers from TextableUtils. Corresponding
# interface elements are declared here and actually drawn below (at
# their position in the UI)...
self.infoBox = InfoBox(widget=self.controlArea)
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute="infoBox",
sendIfPreCallback=None,
)
# User interface...
# Tabs...
self.tabs = QTabWidget()
self.optionsTab = QWidget()
self.modelManagerTab = QWidget()
self.tabs.addTab(self.optionsTab, "Options")
self.tabs.addTab(self.modelManagerTab, "Model manager")
# Options tab...
OptionsTabBox = QHBoxLayout()
optionsBox = gui.widgetBox(widget=self.optionsTab)
self.modelComboBox = gui.comboBox(
widget=optionsBox,
master=self,
value='model',
label='Model: ',
tooltip='Select the spaCy language model you want to use.',
items=INSTALLED_MODELS,
sendSelectedValue=True,
callback=self.modelChanged,
)
OptionsTabBox.addWidget(optionsBox)
self.optionsTab.setLayout(OptionsTabBox)
# Model manager tab...
modelManagerTabBox = QHBoxLayout()
modelManagerBox = gui.widgetBox(widget=self.modelManagerTab)
# TODO: Model manager UI
modelManagerTabBox.addWidget(modelManagerBox)
self.modelManagerTab.setLayout(modelManagerTabBox)
self.controlArea.layout().addWidget(self.tabs)
gui.rubber(self.controlArea)
# Now Info box and Send button must be drawn...
self.sendButton.draw()
self.infoBox.draw()
self.infoBox.setText("Widget needs input", "warning")
# Load spaCy language model...
self.modelChanged()
# Send data if autoSend.
self.sendButton.sendIf()
class SpaCy(OWTextableBaseWidget):
"""Textable widget for NLP using spaCy."""
#----------------------------------------------------------------------
# Widget's metadata...
name = "spaCy"
description = "Natural language processing using spaCy"
icon = "icons/spacy.svg"
priority = 21 # TODO
#----------------------------------------------------------------------
# Channel definitions...
inputs = [("Text data", Segmentation, "inputData")]
outputs = [
("Tokenized text", Segmentation, widget.Default),
("Named entities", Segmentation),
("Noun chunks", Segmentation),
("Sentences", Segmentation),
]
#----------------------------------------------------------------------
# Layout parameters...
want_main_area = False
#----------------------------------------------------------------------
# Settings...
settingsHandler = VersionedSettingsHandler(
version=__version__.rsplit(".", 1)[0]
)
maxLen = settings.Setting("1000000")
annotatePOSTags = settings.Setting(False)
annotateDependencies = settings.Setting(False)
annotateEntities = settings.Setting(False)
segmentEntities = settings.Setting(False)
segmentChunks = settings.Setting(False)
segmentSentences = settings.Setting(False)
autoSend = settings.Setting(False)
model = settings.Setting("")
def __init__(self):
"""Widget creator."""
super().__init__()
if INSTALLED_MODELS:
self.model = INSTALLED_MODELS[0]
else:
self.model = ""
# Other attributes...
self.inputSeg = None
self.nlp = None
self.selectedModels = list()
self.downloadableModelLabels = list()
self.loadedComponents = list()
self.mustLoad = True
# Next two instructions are helpers from TextableUtils. Corresponding
# interface elements are declared here and actually drawn below (at
# their position in the UI)...
self.infoBox = InfoBox(widget=self.controlArea)
self.sendButton = SendButton(
widget=self.controlArea,
master=self,
callback=self.sendData,
infoBoxAttribute="infoBox",
sendIfPreCallback=None,
)
# User interface...
# Tabs...
self.tabs = QTabWidget()
self.optionsTab = QWidget()
self.modelManagerTab = QWidget()
self.tabs.addTab(self.optionsTab, "Options")
self.tabs.addTab(self.modelManagerTab, "Model manager")
# Options tab...
OptionsTabBox = QHBoxLayout()
optionsBox = gui.widgetBox(widget=self.optionsTab)
self.modelComboBox = gui.comboBox(
widget=optionsBox,
master=self,
value='model',
label='Model: ',
tooltip='Select the spaCy language model you want to use.',
items=INSTALLED_MODELS[:],
sendSelectedValue=True,
callback=self.modelComboboxChanged,
)
gui.separator(widget=optionsBox, height=3)
annotationsBox = gui.widgetBox(
widget=optionsBox,
box="Additional token annotations:",
)
annotationsBoxLine1 = gui.widgetBox(
widget=annotationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=annotationsBoxLine1,
master=self,
value='annotatePOSTags',
label='part-of-speech tags',
callback=self.updateDisabledComponents,
tooltip=("Annotate output tokens with part-of-speech tags."),
)
self.annotatePOSTagsReloadLabel = gui.label(
annotationsBoxLine1,
master=self,
label="(reload needed)",
)
self.annotatePOSTagsReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
annotationsBoxLine2 = gui.widgetBox(
widget=annotationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=annotationsBoxLine2,
master=self,
value='annotateDependencies',
label='syntactic dependencies',
callback=self.updateDisabledComponents,
tooltip=("Annotate output tokens with syntactic dependencies."),
)
self.annotateDependenciesReloadLabel = gui.label(
annotationsBoxLine2,
master=self,
label="(reload needed)",
)
self.annotateDependenciesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
annotationsBoxLine3 = gui.widgetBox(
widget=annotationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=annotationsBoxLine3,
master=self,
value='annotateEntities',
label='named entities',
callback=self.updateDisabledComponents,
tooltip=("Annotate output tokens with named entities."),
)
self.annotateEntitiesReloadLabel = gui.label(
annotationsBoxLine3,
master=self,
label="(reload needed)",
)
self.annotateEntitiesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
segmentationsBox = gui.widgetBox(
widget=optionsBox,
box="Additional segmentations:",
)
segmentationsBoxLine1 = gui.widgetBox(
widget=segmentationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=segmentationsBoxLine1,
master=self,
value='segmentEntities',
label='named entities',
callback=self.updateDisabledComponents,
tooltip="Output named entity segmentation on separate channel.",
)
self.segmentEntitiesReloadLabel = gui.label(
segmentationsBoxLine1,
master=self,
label="(reload needed)",
)
self.segmentEntitiesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
segmentationsBoxLine2 = gui.widgetBox(
widget=segmentationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=segmentationsBoxLine2,
master=self,
value='segmentChunks',
label='noun chunks',
callback=self.updateDisabledComponents,
tooltip="Output noun chunk segmentation on separate channel.",
)
self.segmentChunksReloadLabel = gui.label(
segmentationsBoxLine2,
master=self,
label="(reload needed)",
)
self.segmentChunksReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
segmentationsBoxLine3 = gui.widgetBox(
widget=segmentationsBox,
orientation="horizontal",
box=None,
)
gui.checkBox(
widget=segmentationsBoxLine3,
master=self,
value='segmentSentences',
label='sentences',
callback=self.updateDisabledComponents,
tooltip="Output sentence segmentation on separate channel.",
)
self.segmentSentencesReloadLabel = gui.label(
segmentationsBoxLine3,
master=self,
label="(reload needed)",
)
self.segmentSentencesReloadLabel.setStyleSheet(
"font-style: oblique; color: gray"
)
self.updateReloadNeededLabels()
gui.comboBox(
widget=optionsBox,
master=self,
value='maxLen',
items=["1 million"] + ["%i millions" % l for l in range(2, 10)] \
+ ["no limit"],
sendSelectedValue=True,
label=u'Max number of input characters:',
tooltip=(
"The spaCy parser and NER models require roughly 1GB of\n"
"temporary memory per 100'000 characters in the input.\n"
"This means long texts may cause memory allocation errors.\n"
"If you're not using the parser or NER, or have lots of \n"
"RAM, it's probably safe to increase the default limit of\n"
"1 million characters."
),
)
gui.rubber(optionsBox)
OptionsTabBox.addWidget(optionsBox)
self.optionsTab.setLayout(OptionsTabBox)
# Model manager tab...
modelManagerTabBox = QHBoxLayout()
modelManagerBox = gui.widgetBox(widget=self.modelManagerTab)
gui.label(modelManagerBox, self, label="Available models:")
self.downloadableModelsListbox = gui.listBox(
widget=modelManagerBox,
master=self,
value="selectedModels",
labels="downloadableModelLabels",
callback=self.downloadableModelsListboxChanged,
tooltip="Select language models then click Download.",
)
self.downloadableModelsListbox.setSelectionMode(3)
self.downloadableModelLabels = DOWNLOADABLE_MODELS[:]
self.downloadableModelLabels = self.downloadableModelLabels
self.downloadButton = gui.button(
widget=modelManagerBox,
master=self,
label="Download",
callback=self.downloadModels,
tooltip="Download the selected language models.",
)
self.downloadButton.setDisabled(True)
modelManagerTabBox.addWidget(modelManagerBox)
self.modelManagerTab.setLayout(modelManagerTabBox)
self.controlArea.layout().addWidget(self.tabs)
gui.rubber(self.controlArea)
# Now Info box and Send button must be drawn...
self.sendButton.draw()
self.infoBox.draw()
self.infoBox.setText("Widget needs input.", "warning")
# Check that there's a model...
if not self.model:
self.infoBox.setText(
"Please download a language model first.",
"warning",
)
self.tabs.setCurrentIndex(1)
optionsBox.setDisabled(True)
def inputData(self, newInput):
"""Process incoming data."""
self.inputSeg = newInput
self.infoBox.inputChanged()
self.sendButton.sendIf()
def modelComboboxChanged(self):
"""Respond to model change in UI (Options tab)."""
self.mustLoad = True
self.sendButton.settingsChanged()
def downloadableModelsListboxChanged(self):
"""Respond to model change in UI (Model manager tab)."""
self.downloadButton.setDisabled(len(self.selectedModels) == 0)
def downloadModels(self):
"""Respond to Download button (Model manager tab)."""
global INSTALLED_MODELS
# Ask for confirmation...
num_models = len(self.selectedModels)
message = "Your are about to download %i language model@p. " + \
"This may take up to several minutes depending on your " + \
"internet connection. Do you want to proceed?"
message = message % num_models
buttonReply = QMessageBox.question(
self,
"Textable",
pluralize(message, num_models),
QMessageBox.Ok | QMessageBox.Cancel
)
if buttonReply == QMessageBox.Cancel:
return
# Download models...
self.controlArea.setDisabled(True)
progressBar = ProgressBar(self, iterations=num_models)
for model_idx in reversed(self.selectedModels):
model = self.downloadableModelLabels[model_idx]
download_spacy_model(AVAILABLE_MODELS[model])
del self.downloadableModelLabels[model_idx]
progressBar.advance()
# Update GUI...
self.downloadableModelLabels = self.downloadableModelLabels
self.selectedModels = list()
progressBar.finish()
self.controlArea.setDisabled(False)
message = "Downloaded %i language model@p, please restart " + \
"Orange for changes to take effect."
message = message % num_models
QMessageBox.information(
None,
"Textable",
pluralize(message, num_models),
QMessageBox.Ok
)
def updateDisabledComponents(self):
"""Load components if needed."""
self.updateReloadNeededLabels()
self.sendButton.settingsChanged()
def updateReloadNeededLabels(self):
"""Update the labels that indicate whether model reload is needed."""
self.annotatePOSTagsReloadLabel.setVisible(
self.annotatePOSTags and ("tagger" not in self.loadedComponents)
)
self.annotateDependenciesReloadLabel.setVisible(
self.annotateDependencies and ("parser" not in self.loadedComponents)
)
self.annotateEntitiesReloadLabel.setVisible(
self.annotateEntities and ("ner" not in self.loadedComponents)
)
self.segmentSentencesReloadLabel.setVisible(
self.segmentSentences and "parser" not in self.loadedComponents
)
self.segmentChunksReloadLabel.setVisible(
self.segmentChunks and (
("tagger" not in self.loadedComponents)
or ("parser" not in self.loadedComponents)
)
)
self.segmentEntitiesReloadLabel.setVisible(
self.segmentEntities and "ner" not in self.loadedComponents
)
def getComponentStatus(self):
"""Returns the list of disabled/enabled component based on UI state."""
disabledComponents = list()
enabledComponents = list()
if self.annotatePOSTags or self.segmentChunks:
enabledComponents.append("tagger")
else:
disabledComponents.append("tagger")
if self.annotateDependencies or self.segmentChunks or self.segmentSentences:
enabledComponents.append("parser")
else:
disabledComponents.append("parser")
if self.annotateEntities or self.segmentEntities:
enabledComponents.append("ner")
else:
disabledComponents.append("ner")
return disabledComponents, enabledComponents
def loadModel(self):
"""(Re-)load language model if needed."""
# Initialize progress bar.
self.infoBox.setText(
u"Loading language model, please wait...",
"warning",
)
self.controlArea.setDisabled(True)
progressBar = ProgressBar(self, iterations=1)
disabled, enabled = self.getComponentStatus()
self.nlp = spacy.load(
AVAILABLE_MODELS[self.model],
disable=disabled,
)
self.loadedComponents = enabled
self.updateReloadNeededLabels()
self.mustLoad = False
progressBar.advance()
progressBar.finish()
self.controlArea.setDisabled(False)
def sendData(self):
"""Compute result of widget processing and send to output."""
# Check that there's a model...
if not self.model:
self.infoBox.setText(
"Please download a language model first.",
"warning",
)
self.tabs.setCurrentIndex(1)
return
# Check that there's an input...
if self.inputSeg is None:
self.infoBox.setText("Widget needs input.", "warning")
for channel in [c.name for c in self.outputs]:
self.send(channel, None, self)
return
# Check max length and adjust if needed...
inputLength = sum(len(s.get_content()) for s in self.inputSeg)
if self.maxLen != "no limit":
maxNumChar = int(self.maxLen.split()[0]) * 1000000
if inputLength > maxNumChar:
self.infoBox.setText(
"Input exceeds max number of characters set by user.",
"warning",
)
for channel in [c.name for c in self.outputs]:
self.send(channel, None, self)
return
else:
if inputLength > self.nlp.max_length:
maxNumChar = inputLength
# Load components if needed...
disabled, enabled = self.getComponentStatus()
if self.mustLoad or not(
self.nlp and set(enabled) <= set(self.loadedComponents)
):
self.loadModel()
self.nlp.max_length = maxNumChar
# Initialize progress bar.
self.infoBox.setText(
u"Processing, please wait...",
"warning",
)
self.controlArea.setDisabled(True)
progressBar = ProgressBar(self, iterations=len(self.inputSeg))
tokenSegments = list()
entitySegments = list()
chunkSegments = list()
sentenceSegments = list()
# Process each input segment...
for segment in self.inputSeg:
# NLP analysis...
disabled, _ = self.getComponentStatus()
disabled = [c for c in disabled if c in set(self.loadedComponents)]
with self.nlp.disable_pipes(*disabled):
doc = self.nlp(segment.get_content())
# Get token segments...
tokenSegments.extend(spacyItemsToSegments(doc, segment))
# Get named entity segments...
if self.segmentEntities:
entitySegments.extend(spacyItemsToSegments(doc.ents, segment))
# Get noun chunk segments...
if self.segmentChunks:
chunkSegments.extend(
spacyItemsToSegments(doc.noun_chunks, segment),
)
# Get sentences segments...
if self.segmentSentences:
sentenceSegments.extend(
spacyItemsToSegments(doc.sents, segment),
)
progressBar.advance()
# Build segmentations and send them to output...
tokenSeg = Segmentation(tokenSegments, self.captionTitle + "_tokens")
self.send("Tokenized text", tokenSeg, self)
if self.segmentChunks:
chunkSeg = Segmentation(
chunkSegments,
self.captionTitle + "_chunks",
)
self.send("Noun chunks", chunkSeg, self)
if self.segmentEntities:
entitySeg = Segmentation(
entitySegments,
self.captionTitle + "_entities",
)
self.send("Named entities", entitySeg, self)
if self.segmentSentences:
sentenceSeg = Segmentation(
sentenceSegments,
self.captionTitle + "_sentences",
)
self.send("Sentences", sentenceSeg, self)
# Set status to OK and report data size...
message = "%i token@p" % len(tokenSeg)
message = pluralize(message, len(tokenSeg))
if self.segmentChunks:
message += ", %i chunk@p" % len(chunkSeg)
message = pluralize(message, len(chunkSeg))
if self.segmentEntities:
message += ", %i " % len(entitySeg)
message += "entity" if len(entitySeg) == 1 else "entities"
if self.segmentSentences:
message += ", %i sentence@p" % len(sentenceSeg)
message = pluralize(message, len(sentenceSeg))
message += " sent to output."
last_comma_idx = message.rfind(",")
if last_comma_idx > -1:
message = message[:last_comma_idx] + " and" + \
message[last_comma_idx+1:]
self.infoBox.setText(message)
# Clear progress bar.
progressBar.finish()
self.controlArea.setDisabled(False)
self.sendButton.resetSettingsChangedFlag()
# The following method needs to be copied verbatim in
# every Textable widget that sends a segmentation...
def setCaption(self, title):
if 'captionTitle' in dir(self):
changed = title != self.captionTitle
super().setCaption(title)
if changed:
self.sendButton.settingsChanged()
else:
super().setCaption(title)