def downloadModels(self): """Respond to Download button (Model manager tab).""" global INSTALLED_MODELS # Ask for confirmation... num_models = len(self.selectedModels) message = "Your are about to download %i language model@p. " + \ "This may take up to several minutes depending on your " + \ "internet connection. Do you want to proceed?" message = message % num_models buttonReply = QMessageBox.question(self, "Textable", pluralize(message, num_models), QMessageBox.Ok | QMessageBox.Cancel) if buttonReply == QMessageBox.Cancel: return # Download models... self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=num_models) for model_idx in reversed(self.selectedModels): model = self.downloadableModelLabels[model_idx] download_spacy_model(AVAILABLE_MODELS[model]) del self.downloadableModelLabels[model_idx] progressBar.advance() # Update GUI... self.downloadableModelLabels = self.downloadableModelLabels self.selectedModels = list() progressBar.finish() self.controlArea.setDisabled(False) message = "Downloaded %i language model@p, please restart " + \ "Orange for changes to take effect." message = message % num_models QMessageBox.information(None, "Textable", pluralize(message, num_models), QMessageBox.Ok)
def sendData(self): """Compute result of widget processing and send to output""" # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input", "warning") del self.headerList[:] self.headerList = self.headerList self.send("CSV Segmentation", None, self) return # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.inputSeg)) # Treat... for segment in self.csvSeg: pass progressBar.advance() # Set status to OK and report data size... outputSeg = Segmentation(self.csvSeg, label=self.captionTitle) if len(self.contentIsNone) == 0 : message = "%i segment@p sent to output." % len(outputSeg) message = pluralize(message, len(outputSeg)) self.infoBox.setText(message) # message if one or more segments has no content and has been ignored elif len(self.contentIsNone) == 1: message = "%i segment@p sent to output. (ignored %i segment with \ no content)" % (len(outputSeg), len(self.contentIsNone)) message = pluralize(message, len(outputSeg)) self.infoBox.setText(message) else : message = "%i segment@p sent to output. (ignored %i segments with \ no content)" % (len(outputSeg), len(self.contentIsNone)) message = pluralize(message, len(outputSeg)) self.infoBox.setText(message) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Send data to output... self.send("CSV Segmentation", outputSeg, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): if not self.file: self.infoBox.setText(u"Please select input file.", "warning") self.send('Text data', None, self) return # Clear created Inputs. self.clearCreatedInputs() # Get transcription try: transcription = self.get_large_audio_transcription( self.file, language=self.language, set_silence_len=self.selected_dur, set_silence_threshold=self.selected_vol) except speech_recognition.UnknownValueError as err: self.infoBox.setText( u"You seem to have overuseed the built-in API key, refer to the documentation for further informations.", "warning") self.send('Text data', None, self) return # Checks if there is a transcription if transcription is None: self.infoBox.setText(u"You must use mp3 or wav audio files.", "warning") self.send('Text data', None, self) return # Regex to get the name of the input file title = self.file regex = re.compile("[^(/\\)]+[mp3|wav]$") match = re.findall(regex, title) if self.selected_seg: for chunk in transcription: new_input = Input(chunk, label=match) self.createdInputs.append(new_input) else: new_input = Input(transcription, label=match) self.createdInputs.append(new_input) # Concatenates the segmentations in the output segmentation self.segmentation = Segmenter.concatenate( segmentations=self.createdInputs, label=self.captionTitle, copy_annotations=False, import_labels_as="") #Sending segments length message = " Succesfully transcripted ! % i segment@p sent to output" % len( self.segmentation) message = pluralize(message, len(self.segmentation)) # Send token... self.send("Text data", self.segmentation, self) self.infoBox.setText(message) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output.""" # Check that there's a model if not self.model: self.noLanguageModelWarning() return # Check that there's an input if self.inputSeg is None: self.infoBox.setText("Widget needs input.", "warning") self.send('Summary', None, self) self.send('HTML_Summary', None, self) return # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) self.controlArea.setDisabled(True) # Type of segmentation (per segment or per segmentation) segments = list() html_segments = list() if self.typeSeg == "Summarize each segments individually": # Process each segment separately, then create segmentation for segment in self.inputSeg: content = segment.get_content() resume, html_resume = self.summarize(self.cv, content) segments.append(Segment(str_index=resume[0].str_index, )) html_segments.append( Segment(str_index=html_resume[0].str_index, )) elif self.typeSeg == "Summarize all segments as one": merged_seg = " ".join( [segment.get_content() for segment in self.inputSeg]) resume, html_resume = self.summarize(self.cv, merged_seg) segments.append(Segment(str_index=resume[0].str_index, )) html_segments.append(Segment(str_index=html_resume[0].str_index, )) # Create segmentation from segment() and assign it to the output self.outputSeg = Segmentation(segments, self.captionTitle) self.html_outputSeg = Segmentation(html_segments, self.captionTitle) # Send segmentation to output channels self.send("Summary", self.outputSeg, self) self.send('HTML_Summary', self.html_outputSeg, self) # Set message to sent message = "%i segment@p sent to output " % len(self.outputSeg) message = pluralize(message, len(self.outputSeg)) self.infoBox.setText(message) self.sendButton.resetSettingsChangedFlag() self.controlArea.setDisabled(False)
def sendData(self): """Compute result of widget processing and send to output""" # An input is needed if self.inputSeg == None: self.infoBox.setText( "A segmentation input is needed.", "warning" ) self.send("Segmentation with annotations", None, self) return # Skip if no list is selected if self.titleLabels == None: self.infoBox.setText( "You need to define at least one lexical list.", "error" ) self.send("Segmentation with annotations", None, self) return # A list must have been selected if len(self.selectedFields) == 0: self.infoBox.setText( "Please select one or more lexical lists.", "warning" ) self.send("Segmentation with annotations", None, self) return # A annotation key must have been defined """ if self.labelName == "": self.infoBox.setText( "An annotation key is needed.", "warning" ) self.send("Segmentation with annotations", None, self) return """ self.huntTheLexic() # Set status to OK and report data size... message = "%i segment@p sent to output " % len(self.outputSeg) message = pluralize(message, len(self.outputSeg)) # Segmentation go to outputs... self.send("Segmentation with annotations", self.outputSeg, self) self.infoBox.setText(message) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input", "warning") self.send("Linguistically analyzed data", None, self) return # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.inputSeg)) # Basic NLP analysis for dev purposes... analyzedSegments = list() for segment in self.inputSeg: analyzedString = "" doc = self.nlp(segment.get_content()) for token in doc: analyzedString += "%s\t%s\n" % (token.text, token.pos_) analyzedSegments.append(Input(analyzedString)) progressBar.advance() outputSeg = LTTL.Segmenter.concatenate( analyzedSegments, import_labels_as=None, label=self.captionTitle, ) # Set status to OK and report data size... message = "%i segment@p sent to output." % len(outputSeg) message = pluralize(message, len(outputSeg)) self.infoBox.setText(message) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Send data to output... self.send("Linguistically analyzed data", outputSeg, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Skip if title list is empty: if self.titleLabels == list(): return # Check that something has been selected... if len(self.selectedTitles) == 0: self.infoBox.setText("Please select one or more titles.", "warning") self.send("XML-TEI data", None, self) return # Clear created Inputs. self.clearCreatedInputs() # Initialize progress bar. progressBar = gui.ProgressBar(self, iterations=len(self.selectedTitles)) # Attempt to connect to ECP and retrieve plays... xml_contents = list() annotations = list() try: for title in self.selectedTitles: doc_url = self.document_base_url + \ self.filteredTitleSeg[title].annotations["url"] print(doc_url) url = re.sub(r"/([^/]+)\.shtml", r"/\1/\1.xml", doc_url) print(url) response = urllib.request.urlopen(url) xml_contents.append(response.read().decode('utf-8')) source_annotations = \ self.filteredTitleSeg[title].annotations.copy() #source_annotations["url"] = source_annotations["href"] #del source_annotations["href"] annotations.append(source_annotations) progressBar.advance() # 1 tick on the progress bar... # If an error occurs (e.g. http error, or memory error)... except: #Set Info box and widget to "error" state. self.infoBox.setText("Couldn't download data from ECP website.", "error") # Reset output channel. self.send("XML-TEI data", None, self) return # Store downloaded XML in input objects... for xml_content_idx in range(len(xml_contents)): newInput = Input(xml_contents[xml_content_idx], self.captionTitle) self.createdInputs.append(newInput) # If there"s only one play, the widget"s output is the created Input. if len(self.createdInputs) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget"s output is a concatenation... else: self.segmentation = Segmenter.concatenate( self.createdInputs, self.captionTitle, import_labels_as=None, ) # Annotate segments... for idx, segment in enumerate(self.segmentation): segment.annotations.update(annotations[idx]) self.segmentation[idx] = segment # Store imported URLs as setting. self.importedURLs = [ self.filteredTitleSeg[self.selectedTitles[0]].annotations["url"] ] # Set status to OK and report data size... message = "%i segment@p sent to output " % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += "(%i character@p)." % numChars message = pluralize(message, numChars) self.infoBox.setText(message) progressBar.finish() # Clear progress bar. progressBar.finish() # Send token... self.send("XML-TEI data", self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): # Clear created Inputs... self.clearCreatedInputs() if not self.TreetaggerPath: self.infoBox.setText(self.noTreetaggerPathWarning, "warning") self.send("Tagged data", None) return elif not self.getAvailableLanguages(): self.infoBox.setText(self.noLanguageParameterWarning, "warning") self.send("Tagged data", None) return elif not self.segmentation: self.infoBox.setText(u"Widget needs input", "warning") self.send("Tagged data", None) return # Initialize progress bar. self.infoBox.setText(u"Processing, please wait...", "warning") self.controlArea.setDisabled(True) self.progressBar = ProgressBar(self, iterations=5) # Create a copy of input seg, storing annotations in temp attr... copy_of_input_seg = Segmentation() copy_of_input_seg.label = self.segmentation.label for seg_idx, segment in enumerate(self.segmentation): attr = " ".join([ "%s=%s" % ( ''.join(c for c in unicodedata.normalize('NFD', item[0]) if unicodedata.category(c) != 'Mn'), quoteattr(str(item[1])), ) for item in segment.annotations.items() ]) segment.annotations["tt_ax"] = attr copy_of_input_seg.append(segment) self.progressBar.advance() # Dump segmentation in unique string to avoid multiple calls to TT... concatenated_text = copy_of_input_seg.to_string( formatting="<ax_tt %(tt_ax)s>%(__content__)s</ax_tt>", display_all=True, ) self.progressBar.advance() # Tag the segmentation contents... tagopt = '-token -lemma -sgml -quiet' if self.replaceUnknown: tagopt += " -no-unknown" tagger = treetaggerwrapper.TreeTagger( TAGLANG=pycountry.languages.get(name=self.language).alpha_2, TAGOPT=tagopt, TAGDIR=self.TreetaggerPath, ) tagged_lines = tagger.tag_text( concatenated_text, notagurl=True, notagemail=True, notagip=True, notagdns=True, ) tagged_input = Input("\n".join(tagged_lines)) self.createdInputs.append(tagged_input) # Replace <unknown> with [unknown] and " with " then # re-segment to match the original segmentation structure. tagged_segmentation, _ = Segmenter.recode( tagged_input, substitutions=[ (re.compile(r"<unknown>"), "[unknown]"), (re.compile(r'"""'), '"""'), ], ) tagged_segmentation = Segmenter.import_xml(tagged_segmentation, "ax_tt") self.progressBar.advance() # Place each output line of Treetagger in an xml tag with annotations.. xml_segmentation, _ = Segmenter.recode( tagged_segmentation, substitutions=[ (re.compile(r"(.+)\t(.+)\t(.+?)(?=[\r\n])"), '<w lemma="&3" pos-tag="&2">&1</w>'), (re.compile(r'^\n|\n$'), ''), ], ) # Segment into individual tokens if XML output option is disabled... if self.outputFormat == "add XML tags": output_segmentation = xml_segmentation else: try: output_segmentation = Segmenter.import_xml( xml_segmentation, "w") except ValueError: self.infoBox.setText( "Please check that either the input contains well-formed " "XML, or it doesn't contain instances of '<' and '\x3e'", "error") self.send("Tagged data", None) self.progressBar.finish() self.controlArea.setDisabled(False) return self.progressBar.finish() self.controlArea.setDisabled(False) output_segmentation.label = self.captionTitle message = u'%i segment@p sent to output.' % len(output_segmentation) message = pluralize(message, len(output_segmentation)) self.infoBox.setText(message) self.send('Tagged data', output_segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Send data from website springfieldspringfield""" # Skip if title list is empty: if self.myBasket == list(): self.infoBox.setText( "Your corpus is empty, please add some movies first", "warning" ) self.segmentation = None self.send("Movie transcripts", self.segmentation, self) return # Clear created Inputs. self.clearCreatedInputs() annotations = list() script_list = list() annotations_dict = dict() self.controlArea.setDisabled(True) # Initialize progress bar. progressBar = ProgressBar(self, iterations=len(self.myBasket)) # This part of code is what fetches the actual script try: for movie in self.myBasket: # Each movie that is in the corpus is split into title and year # (rsplit makes sure to only split last occurence) which will # become annotations b = copy.copy(movie) future_annotation = b.rsplit('(', 1) movie_title = future_annotation[0] movie_year = future_annotation[-1] movie_year = movie_year[:-1] annotations_dict["Movie Title"] = movie_title annotations_dict["Year of release"] = movie_year # It is important to make a copy of dictionary, otherwise each # iteration will replace every element of the annotations list annotations.append(annotations_dict.copy()) # link_end and page_url are the two variables that will have to # be changed in case scripts need to be taken from elsewhere link_end = self.path_storage[movie] page_url = "https://www.springfieldspringfield.co.uk/" + \ "movie_script.php?movie=" + link_end page = urllib.request.urlopen(page_url) soup = BeautifulSoup(page, 'html.parser') # This is what grabs the movie script script = soup.find("div", {"class":"movie_script"}) script_list.append(script.text) # 1 tick on the progress bar of the widget progressBar.advance() except: self.infoBox.setText( "Couldn't download data from SpringfieldSpringfield website.", "error" ) self.controlArea.setDisabled(False) return # Store downloaded script strings in input objects... for script in script_list: newInput = Input(script, self.captionTitle) self.createdInputs.append(newInput) # If there's only one play, the widget"s output is the created Input. if len(self.createdInputs) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget"s output is a concatenation... else: self.segmentation = Segmenter.concatenate( self.createdInputs, self.captionTitle, import_labels_as=None, ) # Annotate segments... for idx, segment in enumerate(self.segmentation): segment.annotations.update(annotations[idx]) self.segmentation[idx] = segment # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Set status to OK and report data size... message = "%i segment@p sent to output " % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += "(%i character@p)." % numChars message = pluralize(message, numChars) self.infoBox.setText(message) self.send("Movie transcripts", self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Skip if title list is empty: if self.myBasket == list(): self.infoBox.setText( "Your corpus is empty, please add some books first", "warning") return # Clear created Inputs. self.clearCreatedInputs() self.controlArea.setDisabled(True) # Initialize progress bar. progressBar = ProgressBar( self, iterations=len(self.myBasket), ) text_content = list() annotations = list() try: # Retrieve selected texts from gutenberg for text in self.myBasket: gutenberg_id = text[2] # Get the text with Gutenbergpy gutenberg_text = gutenbergpy.textget.strip_headers( gutenbergpy.textget.get_text_by_id(gutenberg_id)).decode( "utf-8") text_content.append(gutenberg_text) # populate the annotation list annotations.append([text[0], text[1], text[3]]) progressBar.advance() # If an error occurs (e.g. http error, or memory error)... except Exception as exc: # Set Info box and widget to "error" state. self.infoBox.setText("Couldn't download data from Gutenberg", "error") self.controlArea.setDisabled(False) print(exc) return # Store downloaded text strings in input objects... for text in text_content: newInput = Input(text, self.captionTitle) self.createdInputs.append(newInput) # If there's only one text, the widget's output is the created Input. if len(self.createdInputs) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget"s output is a concatenation. else: self.segmentation = Segmenter.concatenate( self.createdInputs, self.captionTitle, import_labels_as=None, ) # Annotate segments with book metadata for idx, segment in enumerate(self.segmentation): segment.annotations.update({"title": annotations[idx][0]}) segment.annotations.update({"author": annotations[idx][1]}) segment.annotations.update({"language": annotations[idx][2]}) self.segmentation[idx] = segment # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Set status to OK and report data size... message = "%i segment@p sent to output " % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += "(%i character@p)." % numChars message = pluralize(message, numChars) self.infoBox.setText(message) self.send("Gutenberg importation", self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Skip if title list is empty: if self.myBasket == list(): self.infoBox.setText( "Your corpus is empty, please add some books first", "warning") return # Clear created Inputs. self.clearCreatedInputs() self.controlArea.setDisabled(True) # Initialize progress bar. progressBar = ProgressBar( self, iterations=len(self.myBasket), ) selectedTexts = list() text_content = list() annotations = list() # get the Gutenberg cache cache = GutenbergCache.get_cache() try: # TODO: Retrieve selected texts from gutenberg for text in self.myBasket: # Get the id of the text query_id = cache.native_query( sql_query= "select gutenbergbookid from books where id == {selected_id}" .format(selected_id=text[2])) gutenberg_id = list(query_id) # Get the text with Gutenbergpy gutenberg_text = gutenbergpy.textget.strip_headers( gutenbergpy.textget.get_text_by_id(gutenberg_id[0][0])) text_content.append(gutenberg_text) annotations.append(text[1]) progressBar.advance() # If an error occurs (e.g. http error, or memory error)... except Exception: # Set Info box and widget to "error" state. self.infoBox.setText("Couldn't download data from Gutenberg", "error") self.controlArea.setDisabled(False) return # TODO: send gutenberg texts as output # Store downloaded lyrics strings in input objects... for text in text_content: newInput = Input(text, self.captionTitle) self.createdInputs.append(newInput) # If there"s only one play, the widget"s output is the created Input. if len(self.createdInputs) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget"s output is a concatenation... else: self.segmentation = Segmenter.concatenate( self.createdInputs, self.captionTitle, import_labels_as=None, ) # TODO: annotate with book metadata # Annotate segments... for idx, segment in enumerate(self.segmentation): segment.annotations.update({"title": annotations[idx]}) self.segmentation[idx] = segment # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Set status to OK and report data size... message = "%i segment@p sent to output " % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += "(%i character@p)." % numChars message = pluralize(message, numChars) self.infoBox.setText(message) self.send("Gutenberg importation", self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Load folders, create and send segmentation""" # Check that there's something on input... if (self.displayAdvancedSettings and not self.folders) or not (self.rootFolderPath or self.displayAdvancedSettings): self.infoBox.setText(u'Please select input folder.', 'warning') self.send('Text data', None, self) return # Check that autoNumberKey is not empty (if necessary)... if self.displayAdvancedSettings and self.autoNumber: if self.autoNumberKey: autoNumberKey = self.autoNumberKey else: self.infoBox.setText( u'Please enter an annotation key for auto-numbering.', 'warning') self.send('Text data', None, self) return else: autoNumberKey = None # Clear created Inputs... self.clearCreatedInputs() annotations = list() counter = 1 if self.displayAdvancedSettings: myFolders = self.folders else: myFolders = [self.folder] # Annotations... allFileListContent = list() for myFolder in myFolders: myFiles = myFolder['fileList'] for myFile in myFiles: annotation = dict() annotation['file name'] = myFile['fileName'] annotation['file depth level'] = myFile['depthLvl'] annotation['file path'] = myFile['absoluteFilePath'] try: annotation['file encoding, confidence'] = myFile[ 'encoding'] + ", " + str(myFile['encodingConfidence']) except TypeError: annotation['file encoding, confidence'] = "unknown" depths = [k for k in myFile.keys() if k.startswith('depth_')] for depth in depths: annotation[depth] = myFile[depth] annotations.append(annotation) allFileListContent.append(myFile['fileContent']) # Create an LTTL.Input for each files... if len(allFileListContent) == 1: label = self.captionTitle else: label = None for index in range(len(allFileListContent)): myInput = Input(allFileListContent[index], label) segment = myInput[0] segment.annotations.update(annotations[index]) myInput[0] = segment self.createdInputs.append(myInput) # If there's only one file, the widget's output is the created Input. if len(allFileListContent) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget's output is a concatenation... else: self.segmentation = Segmenter.concatenate( segmentations=self.createdInputs, label=self.captionTitle, copy_annotations=True, import_labels_as=None, sort=False, auto_number_as=None, merge_duplicates=False, progress_callback=None, ) message = u'%i segment@p sent to output ' % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += u'(%i character@p).' % numChars message = pluralize(message, numChars) self.infoBox.setText(message) self.send('Text data', self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output.""" # Check that there's a model... if not self.model: self.infoBox.setText( "Please download a language model first.", "warning", ) self.tabs.setCurrentIndex(1) return # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input.", "warning") for channel in [c.name for c in self.outputs]: self.send(channel, None, self) return # Check max length and adjust if needed... inputLength = sum(len(s.get_content()) for s in self.inputSeg) if self.maxLen != "no limit": maxNumChar = int(self.maxLen.split()[0]) * 1000000 if inputLength > maxNumChar: self.infoBox.setText( "Input exceeds max number of characters set by user.", "warning", ) for channel in [c.name for c in self.outputs]: self.send(channel, None, self) return else: if inputLength > self.nlp.max_length: maxNumChar = inputLength # Load components if needed... disabled, enabled = self.getComponentStatus() if self.mustLoad or not( self.nlp and set(enabled) <= set(self.loadedComponents) ): self.loadModel() self.nlp.max_length = maxNumChar # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.inputSeg)) tokenSegments = list() entitySegments = list() chunkSegments = list() sentenceSegments = list() # Process each input segment... for segment in self.inputSeg: # NLP analysis... disabled, _ = self.getComponentStatus() disabled = [c for c in disabled if c in set(self.loadedComponents)] with self.nlp.disable_pipes(*disabled): doc = self.nlp(segment.get_content()) # Get token segments... tokenSegments.extend(spacyItemsToSegments(doc, segment)) # Get named entity segments... if self.segmentEntities: entitySegments.extend(spacyItemsToSegments(doc.ents, segment)) # Get noun chunk segments... if self.segmentChunks: chunkSegments.extend( spacyItemsToSegments(doc.noun_chunks, segment), ) # Get sentences segments... if self.segmentSentences: sentenceSegments.extend( spacyItemsToSegments(doc.sents, segment), ) progressBar.advance() # Build segmentations and send them to output... tokenSeg = Segmentation(tokenSegments, self.captionTitle + "_tokens") self.send("Tokenized text", tokenSeg, self) if self.segmentChunks: chunkSeg = Segmentation( chunkSegments, self.captionTitle + "_chunks", ) self.send("Noun chunks", chunkSeg, self) if self.segmentEntities: entitySeg = Segmentation( entitySegments, self.captionTitle + "_entities", ) self.send("Named entities", entitySeg, self) if self.segmentSentences: sentenceSeg = Segmentation( sentenceSegments, self.captionTitle + "_sentences", ) self.send("Sentences", sentenceSeg, self) # Set status to OK and report data size... message = "%i token@p" % len(tokenSeg) message = pluralize(message, len(tokenSeg)) if self.segmentChunks: message += ", %i chunk@p" % len(chunkSeg) message = pluralize(message, len(chunkSeg)) if self.segmentEntities: message += ", %i " % len(entitySeg) message += "entity" if len(entitySeg) == 1 else "entities" if self.segmentSentences: message += ", %i sentence@p" % len(sentenceSeg) message = pluralize(message, len(sentenceSeg)) message += " sent to output." last_comma_idx = message.rfind(",") if last_comma_idx > -1: message = message[:last_comma_idx] + " and" + \ message[last_comma_idx+1:] self.infoBox.setText(message) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" if not self.importedCorpora: self.infoBox.setText("Please add a corpus to the selection.", "warning") self.send("Files", None, self) self.send("Utterances", None, self) return # Clear created Inputs and initialize progress bar... self.clearCreatedInputs() numberOfSteps = 2 if self.outputUtterances else 1 numberOfSteps += 2 if self.outputWords else 0 self.infoBox.setText( "(1/%i) Retrieving data, please wait..." % numberOfSteps, "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.importedCorpora)) annotations = list() # Iterate over corpora... for importedCorpus in self.importedCorpora: corpus = importedCorpus.split("/")[-1] # Try to retrieve corpus from cache... try: basepath = os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))) corpusFilepath = os.path.normpath( os.path.join( basepath, self.__class__.cachedFoldername, importedCorpus[len(self.__class__.baseUrl):], )) myZip = zipfile.ZipFile(corpusFilepath) except IOError: # Else try to download (and cache) requested zip file... try: response = requests.get(importedCorpus) myZip = zipfile.ZipFile(io.BytesIO(response.content)) corpusFolderpath = os.path.dirname(corpusFilepath) try: os.makedirs(corpusFolderpath) except OSError: pass try: outputFile = open(corpusFilepath, "wb") outputFile.write(response.content) outputFile.close() except IOError: pass # If an error occurs (e.g. connection error)... except: # Set Info box and widget to "error" state. self.infoBox.setText( "Couldn't download corpus %s from CHILDES website." % corpus, "error") # Reset output channel. self.send("Files", None, self) self.send("Utterances", None, self) progressBar.finish() self.controlArea.setDisabled(False) return # Create Input for each zipped file and store annotations... for file in myZip.infolist(): file_content = myZip.read(file).decode('utf-8') # If word segmentation is requested... if self.outputWords: # Implement replacements. file_content = re.sub( r"<w.+?(<replacement.+</replacement>).*?</w>", r"\1", file_content, ) # Prepend pre-clitics. file_content, n = re.subn( r"(<mor .+?)(<mor-pre>.+</mor-pre>)", r"\2\1", file_content, ) # Move <gra> into <mw>. file_content, n = re.subn( r"(</mw>)(<gra.+?/>)", r"\2\1", file_content, ) newInput = Input(file_content, self.captionTitle + "_files") self.createdInputs.append(newInput) chatSeg = Segmenter.import_xml(newInput, "CHAT") annotations.append(dict()) annotations[-1]["file_path"] = file.filename for key in ["Corpus", "Lang", "PID"]: try: annotations[-1][key.lower()] = \ chatSeg[0].annotations[key] except KeyError: pass participantListSeg = Segmenter.import_xml( newInput, "Participants") recodedInput, _ = Segmenter.recode( participantListSeg, [(re.compile("/>"), "> </participant>")]) participantSeg = Segmenter.import_xml(recodedInput, "participant") targetChildData = list() for participant in participantSeg: if participant.annotations["role"] != "Target_Child": continue targetChildData.append(dict()) if "age" in participant.annotations: targetChildData[-1]["target_child_age"] = \ participant.annotations["age"] age_parse = re.search( r"(\d+)Y(\d+)M(\d+)D", participant.annotations["age"], ) if age_parse: targetChildData[-1]["target_child_years"] = \ age_parse.group(1) months = int(age_parse.group(2)) \ + 12 * int(age_parse.group(1)) targetChildData[-1]["target_child_months"] = \ '%02d' % months days = int(age_parse.group(3)) \ + 30 * months targetChildData[-1]["target_child_days"] = \ '%02d' % days if "id" in participant.annotations: targetChildData[-1]["target_child_id"] = \ participant.annotations["id"] if "sex" in participant.annotations: targetChildData[-1]["target_child_sex"] = \ participant.annotations["sex"] if len(targetChildData) == 1: annotations[-1].update(targetChildData[0]) progressBar.advance() # If there's only one file, the widget's output is the created Input... if len(self.createdInputs) == 1: self.fileSegmentation = self.createdInputs[0] # Otherwise the widget's output is a concatenation... else: self.fileSegmentation = Segmenter.concatenate( self.createdInputs, self.captionTitle + "_files", import_labels_as=None, ) # Annotate segments... for idx, segment in enumerate(self.fileSegmentation): segment.annotations.update(annotations[idx]) self.fileSegmentation[idx] = segment # Terminate progress bar... progressBar.finish() message = "%i file@p" % len(self.fileSegmentation) message = pluralize(message, len(self.fileSegmentation)) self.send("Files", self.fileSegmentation, self) # Build utterance segmentation if needed... if self.outputUtterances: self.infoBox.setText( "(2/%i) Building utterance segmentation, please wait..." \ % numberOfSteps, "warning", ) progressBar = ProgressBar(self, iterations=len(self.fileSegmentation)) self.utteranceSegmentation = Segmenter.import_xml( self.fileSegmentation, "u", progress_callback=progressBar.advance, label=self.captionTitle + "_utterances", ) progressBar.finish() message += " and " if not self.outputWords else ", " message += "%i utterance@p" % len(self.utteranceSegmentation) message = pluralize(message, len(self.utteranceSegmentation)) self.send("Utterances", self.utteranceSegmentation, self) else: self.send("Utterances", None, self) # Build word segmentation if needed... if self.outputWords: self.infoBox.setText( "(%i/%i) Building word segmentation, please wait..." \ % (2 + (1 if self.outputUtterances else 0), numberOfSteps), "warning", ) try: baseSegmentation = self.utteranceSegmentation except: baseSegmentation = self.fileSegmentation progressBar = ProgressBar(self, iterations=2 * len(baseSegmentation)) wordSegmentation = Segmenter.import_xml( baseSegmentation, "w", progress_callback=progressBar.advance, ) mwSegmentation = Segmenter.import_xml( baseSegmentation, "mw", progress_callback=progressBar.advance, ) # Analyze words to extract annotations... self.infoBox.setText( "(%i/%i) Extracting word annotations, please wait..." \ % (3 + (1 if self.outputUtterances else 0), numberOfSteps), "warning", ) progressBar.finish() progressBar = ProgressBar(self, iterations=len(wordSegmentation)) wordSegments = list() for word in wordSegmentation: mws = word.get_contained_segments(mwSegmentation) if mws: for mw in mws: wordSegment = word.deepcopy() wordSegment.annotations.update( self.extractWordAnnotations(mw)) wordSegments.append(wordSegment) else: wordSegments.append(word) progressBar.advance() self.wordSegmentation = Segmentation( wordSegments, label=self.captionTitle + "_words", ) message += " and %i word@p" % len(self.wordSegmentation) message = pluralize(message, len(self.wordSegmentation)) self.send("Words", self.wordSegmentation, self) else: self.send("Words", None, self) # Set status to OK and report data size... message += " sent to output." message = pluralize(message, len(self.fileSegmentation)) self.infoBox.setText(message) progressBar.finish() self.controlArea.setDisabled(False) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Load folders, create and send segmentation""" # Check that there's something on input... if ( (self.displayAdvancedSettings and not self.folders) or not (self.rootFolderPath or self.displayAdvancedSettings) ): self.infoBox.setText(u'Please select input folder.', 'warning') self.send('Text data', None, self) return # Check that autoNumberKey is not empty (if necessary)... if self.displayAdvancedSettings and self.autoNumber: if self.autoNumberKey: autoNumberKey = self.autoNumberKey else: self.infoBox.setText( u'Please enter an annotation key for auto-numbering.', 'warning' ) self.send('Text data', None, self) return else: autoNumberKey = None # Clear created Inputs... self.clearCreatedInputs() fileContents = list() annotations = list() counter = 1 if self.displayAdvancedSettings: myFolders = self.folders else: myFolders = [[self.rootFolderPath]] progressBar = gui.ProgressBar( self, iterations=len(myFolders) ) # Walk through each folder and open each files successively... fileContents = self.fileContents # Annotations... myFolders = self.folders for myFolder in myFolders: myFiles = myFolder['fileList'] for myFile in myFiles: # print(myFile) annotation = dict() if self.importFileNameKey: annotation[self.importFileNameKey] = myFile['fileName'] if self.importFolderNameKey: annotation[self.importFolderNameKey] = myFile['folderName'] if self.FolderDepth1Key: annotation[self.FolderDepth1Key] = myFile['depth1'] if self.FolderDepth2Key: annotation[self.FolderDepth2Key] = myFile['depth2'] if self.FolderDepthLvl: annotation[self.FolderDepthLvl] = myFile['depthLvl'] annotations.append(annotation) # progressBar.advance() # Create an LTTL.Input for each files... if len(fileContents) == 1: label = self.captionTitle else: label = None for index in range(len(fileContents)): myInput = Input(fileContents[index], label) segment = myInput[0] segment.annotations.update(annotations[index]) myInput[0] = segment self.createdInputs.append(myInput) # If there's only one file, the widget's output is the created Input. if len(fileContents) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget's output is a concatenation... else: self.segmentation = Segmenter.concatenate( segmentations=self.createdInputs, label=self.captionTitle, copy_annotations=True, import_labels_as=None, sort=False, auto_number_as=None, merge_duplicates=False, progress_callback=None, ) message = u'%i segment@p sent to output ' % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += u'(%i character@p).' % numChars message = pluralize(message, numChars) self.infoBox.setText(message) progressBar.finish() self.send('Text data', self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Clear morphology... self.morphology = dict() # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input", "warning") self.send("Morphologically analyzed data", None, self) self.updateGUI() return # Perform morphological analysis... # Initialize progress bar. self.infoBox.setText( u"Processing, please wait (word count)...", "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=100) # Word count... wordCounts = collections.Counter( [segment.get_content() for segment in self.inputSeg]) self.morphology["wordCounts"] = wordCounts self.infoBox.setText( u"Processing, please wait (signature extraction)...", "warning", ) progressBar.advance(5) # 5 ticks on the progress bar... # Learn signatures... try: lxa5crab.crab_nebula.MIN_STEM_LEN = self.minStemLen signatures, stems, suffixes = lxa5crab.find_signatures(wordCounts) self.morphology["signatures"] = signatures self.morphology["stems"] = stems self.morphology["suffixes"] = suffixes except ValueError as e: self.infoBox.setText(e.__str__(), "warning") self.send("Morphologically analyzed data", None, self) self.controlArea.setDisabled(False) progressBar.finish() # Clear progress bar. self.morphology = dict() self.updateGUI() return self.infoBox.setText( u"Processing, please wait (word parsing)...", "warning", ) progressBar.advance(80) # Parse words... parser = lxa5crab.build_parser(wordCounts, signatures, stems, suffixes) self.morphology["parser"] = parser newSegments = list() num_analyzed_words = 0 for segment in self.inputSeg: parses = parser[segment.get_content()] newSegment = segment.deepcopy() if parses[0].signature: num_analyzed_words += 1 newSegment.annotations.update( { "stem": parses[0].stem, "suffix": parses[0].suffix \ if len(parses[0].suffix) else "NULL", "signature": parses[0].signature } ) newSegments.append(newSegment) self.send( "Morphologically analyzed data", Segmentation(newSegments, self.captionTitle), self, ) self.updateGUI() progressBar.advance(15) # Set status to OK and report data size... message = "%i segment@p sent to output (%.2f%% analyzed)." % (len( self.inputSeg), (num_analyzed_words / len(self.inputSeg) * 100)) message = pluralize(message, len(self.inputSeg)) self.infoBox.setText(message) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) self.sendButton.resetSettingsChangedFlag()
def treat_input(self): # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input", "warning") del self.headerList[:] self.headerList = self.headerList return # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.inputSeg)) # clear lists del self.csvSeg[:] del self.contentIsNone[:] # Process each input segment... for segment in self.inputSeg: # Input segment attributes... inputContent = segment.get_content() if not self.deleteQuotes == False : inputContent = inputContent.replace('"',"") inputAnnotations = segment.annotations inputStrIdx = segment.str_index inputStart = segment.start or 0 inputEnd = segment.end or len(inputContent) #Call data processing csv_stream = io.StringIO(inputContent) dialect = sniffer.sniff(csv_stream.readline()) dialect.quoting=csv.QUOTE_NONE csv_stream.seek(0) my_reader = csv.reader(csv_stream, dialect) position = 0 # Process each seg in inputContent for seg in inputContent: segAnnotations = inputAnnotations.copy() # This will launch if sniffer detects a header in the content. if sniffer.has_header(inputContent) == True: # go back to the start otherwise we're going to start from the # second row csv_stream.seek(0) # the header row is defined here. if self.isRenamed == False : self.dict_keys = next(my_reader) for key in self.dict_keys: # this is position of first content # TODO : separator length (if not 1) position += (len(key) + 1) else : input_keys = next(my_reader) for key in input_keys: # this is position of first content # TODO : separator length (if not 1) position += (len(key) + 1) # This will launch if sniffer does not detect a header # in the content. if sniffer.has_header(inputContent) == False: # go back to the start otherwise we're going to start from the # second row. we do this here even though we don't really care # about the first row simply because in general we consider the # first row to not have any missing values csv_stream.seek(0) first_row = next(my_reader) n_cols = len(first_row) if self.isRenamed == False : self.dict_keys = list() for item in range(1, n_cols+1): self.dict_keys.append(str(item)) csv_stream.seek(0) # clear the list before appending del self.headerList[:] for key in self.dict_keys: # appends the headers to the gui list if self.dict_keys.index(key) == self.content_column: self.headerList.append(str(key)+"(*content)") self.headerList = self.headerList else : self.headerList.append(str(key)) self.headerList = self.headerList for idx, row in enumerate(my_reader, start=2): # Get old annotations in new dictionary oldAnnotations = inputAnnotations.copy() segAnnotations = dict() # initiate next row starting position next_position = position for key in oldAnnotations.keys(): segAnnotations[key] = oldAnnotations[key] # This is the main part where we transform our data into # annotations. for key in self.dict_keys: # segAnnotations["length"] = position # segAnnotations["row"] = str(row) # if column is content (first column (0) by default) if self.dict_keys.index(key) == self.content_column: # put value as content content = row[self.dict_keys.index(key)] # else we put value in annotation else: # only if value is not None if len(row[self.dict_keys.index(key)]) != 0 : segAnnotations[key] = row[self.dict_keys.index(key)] # implement position and next_position depending on # content column if self.dict_keys.index(key) < self.content_column: position += len(row[self.dict_keys.index(key)]) + 1 next_position += len(row[self.dict_keys.index(key)]) + 1 if self.dict_keys.index(key) >= self.content_column: next_position += len(row[self.dict_keys.index(key)]) + 1 if len(content) != 0: self.csvSeg.append( Segment( str_index = inputStrIdx, start = position, end = position + len(content), annotations = segAnnotations ) ) else : # if no content, add idx of the row and do not append # TODO : something with contentIsNone self.contentIsNone.append(idx) # initiate new row starting position position = next_position progressBar.advance() unSeg = len(self.csvSeg) # Set status to OK and report segment analyzed... message = "%i segment@p analyzed." % unSeg message = pluralize(message, unSeg) message += " (Ignored %i segment@p with no content)" % \ len(self.contentIsNone) message = pluralize(message, len(self.contentIsNone)) self.infoBox.setText(message) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) self.sendButton.resetSettingsChangedFlag() self.sendButton.sendIf()
def sendData(self): """Compute result of widget processing and send to output""" # Skip if title list is empty: if self.myBasket == list(): self.infoBox.setText( "Your corpus is empty, please add some songs first", "warning") return # Clear created Inputs. self.clearCreatedInputs() self.controlArea.setDisabled(True) # Initialize progress bar. progressBar = ProgressBar(self, iterations=len(self.myBasket)) # Attempt to connect to Genius and retrieve lyrics... selectedSongs = list() song_content = list() annotations = list() try: for song in self.myBasket: # song is a dict {'idx1':{'title':'song1'...}, # 'idx2':{'title':'song2'...}} page_url = "http://genius.com" + song['path'] lyrics = self.html_to_text(page_url) song_content.append(lyrics) annotations.append(song.copy()) # 1 tick on the progress bar of the widget progressBar.advance() # If an error occurs (e.g. http error, or memory error)... except: # Set Info box and widget to "error" state. self.infoBox.setText("Couldn't download data from Genius website.", "error") self.controlArea.setDisabled(False) return # Store downloaded lyrics strings in input objects... for song in song_content: newInput = Input(song, self.captionTitle) self.createdInputs.append(newInput) # If there"s only one play, the widget"s output is the created Input. if len(self.createdInputs) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget"s output is a concatenation... else: self.segmentation = Segmenter.concatenate( self.createdInputs, self.captionTitle, import_labels_as=None, ) # Annotate segments... for idx, segment in enumerate(self.segmentation): segment.annotations.update(annotations[idx]) self.segmentation[idx] = segment # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Set status to OK and report data size... message = "%i segment@p sent to output " % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += "(%i character@p)." % numChars message = pluralize(message, numChars) self.infoBox.setText(message) self.send("Lyrics importation", self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Skip if title list is empty: if self.myBasket == list(): self.infoBox.setText( "Your corpus is empty, please add some movies first", "warning") return # Clear created Inputs. self.clearCreatedInputs() self.controlArea.setDisabled(True) # Initialize progress bar. progressBar = ProgressBar(self, iterations=len(self.myBasket)) # Connect to imdb and add elements in lists list_review = list() list_annotation = list() annotations = list() try: for item in self.myBasket: movie = self.ia.get_movie_reviews(item['id']) movie_annotations = self.ia.get_movie(item['id']) list_review.append(movie) list_annotation.append(movie_annotations) # 1 tick on the progress bar of the widget progressBar.advance() # If an error occurs (e.g. http error, or memory error)... except: # Set Info box and widget to "error" state. self.infoBox.setText("Couldn't download data from imdb", "error") self.controlArea.setDisabled(False) return # Store movie critics strings in input objects... for movie in list_review: data = movie.get('data', "") reviews_data = data.get('reviews') for review in reviews_data: reviews = review.get('content') newInput = Input(reviews) self.createdInputs.append(newInput) for item in list_annotation: print(item) # Store the annotation as dicts in a separate list annotations_dict = {"title": item, "year": item["year"]} annot_dict_copy = annotations_dict.copy() for i in range(25): annotations.append(annot_dict_copy) print(annotations) # If there's only one item, the widget's output is the created Input. if len(self.createdInputs) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget's output is a concatenation... else: self.segmentation = Segmenter.concatenate( self.createdInputs, import_labels_as=None, ) # Annotate segments... for idx, segment in enumerate(self.segmentation): segment.annotations.update(annotations[idx]) self.segmentation[idx] = segment # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Set status to OK and report data size... message = f"{len(self.segmentation)} segment@p sent to output" message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += " (%i character@p)." % numChars message = pluralize(message, numChars) self.infoBox.setText(message) self.send('Segmentation', self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output.""" # Check that there's a model... if not self.model: self.noLanguageModelWarning() self.sendNoneToOutputs() return # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input.", "warning") self.sendNoneToOutputs() return # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) # Disable control area and initialize progress bar... self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.char_df)) # Get start and end pos of concatenated input segments... startPositions = [0] endPositions = list() numSegments = len(self.inputSeg) for idx in range(1, numSegments): prevSegLen = len(self.inputSeg[idx-1].get_content()) startPositions.append(startPositions[-1] + prevSegLen + 1) endPositions.append(startPositions[-1] - 1) endPositions.append(startPositions[-1] + len(self.inputSeg[-1].get_content()) + 1) # Get or update character aliases... find_pairs = sys.modules['charnetto.find_pairs'] characters = [entry.split(", ") for entry in self.characters] find_pairs.map_names(self.char_df, characters) # Initializations... charSegments = list() currentSegmentIdx = 0 # For each character token in Charnetto's output... for index, charToken in self.char_df.iterrows(): # Skip non-PER named entities. if charToken["tag"] != "PER": continue # Get index of containing segment... while charToken["end_pos"] > endPositions[currentSegmentIdx]: currentSegmentIdx += 1 # Create segment for char with its actual coordinates... strIndex = self.inputSeg[currentSegmentIdx].str_index start = charToken["start_pos"]-startPositions[currentSegmentIdx] end = charToken["end_pos"]-startPositions[currentSegmentIdx] annotations = {"id": charToken["alias"]} charSegments.append(Segment(strIndex, start, end, annotations)) progressBar.advance() # Send output... outputSegmentation = Segmentation(charSegments, label=self.captionTitle) self.send("Character segmentation", outputSegmentation, self) print(outputSegmentation.to_string()) # Set status to OK and report data size... message = "%i segment@p sent to output." % len(outputSegmentation) message = pluralize(message, len(outputSegmentation)) self.infoBox.setText(message) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Load files, create and send segmentation""" # Check that there's something on input... if ((self.displayAdvancedSettings and not self.files) or not (self.file or self.displayAdvancedSettings)): self.infoBox.setText(u'Please select input file.', 'warning') self.send('Text data', None, self) return # Check that autoNumberKey is not empty (if necessary)... if self.displayAdvancedSettings and self.autoNumber: if self.autoNumberKey: autoNumberKey = self.autoNumberKey else: self.infoBox.setText( u'Please enter an annotation key for auto-numbering.', 'warning') self.send('Text data', None, self) return else: autoNumberKey = None # Clear created Inputs... self.clearCreatedInputs() fileContents = list() annotations = list() counter = 1 if self.displayAdvancedSettings: myFiles = self.files else: myFiles = [[self.file, self.encoding, "", "", "", "eng", False]] self.infoBox.setText(u"Processing, please wait...", "warning") self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(myFiles)) # Open and process each file successively... for myFile in myFiles: filePath = myFile[0] encoding = myFile[1] encoding = re.sub(r"[ ]\(.+", "", encoding) annotation_key = myFile[2] annotation_value = myFile[3] pdf_password = myFile[4] # SuperTextFiles ocr_languages = myFile[5] # SuperTextFiles ocr_force = myFile[6] # SuperTextFiles myFiletype = filetype.guess(myFile[0]) # SuperTextFiles # Try to open the file... self.error() # Start SuperTextFiles try: if myFiletype is None: fileContent = self.extract_raw_text(filePath, encoding) elif myFiletype.extension == "pdf": if ocr_force is True: fileContent = self.get_pdf_content( filePath, ocr_languages, ) else: if self.is_textual_pdf_file(filePath) is True: fileContent = self.extract_text_from_pdf(filePath) else: fileContent = self.get_pdf_content( filePath, ocr_languages, ) elif myFiletype.extension in IMG_FILETYPES: fileContent = self.ocrize(filePath, ocr_languages) if fileContent == -1: message = u"Couldn't open file." self.infoBox.setText(message, 'error') self.send('Text data', None, self) self.controlArea.setDisabled(False) return # End SuperTextFiles except IOError as e: if "tesseract" in str(e): QMessageBox.warning(None, 'Textable', str(e), QMessageBox.Ok) progressBar.finish() if len(myFiles) > 1: message = u"Couldn't open file '%s'." % filePath else: message = u"Couldn't open file." self.infoBox.setText(message, 'error') self.send('Text data', None, self) self.controlArea.setDisabled(False) return # Remove utf-8 BOM if necessary... if encoding == u'utf-8': fileContent = fileContent.lstrip( codecs.BOM_UTF8.decode('utf-8')) # Normalize text (canonical decomposition then composition)... fileContent = normalize('NFC', fileContent) fileContents.append(fileContent) # Annotations... annotation = dict() if self.displayAdvancedSettings: if annotation_key and annotation_value: annotation[annotation_key] = annotation_value if self.importFilenames and self.importFilenamesKey: filename = os.path.basename(filePath) annotation[self.importFilenamesKey] = filename if self.autoNumber and self.autoNumberKey: annotation[self.autoNumberKey] = counter counter += 1 annotations.append(annotation) progressBar.advance() # Create an LTTL.Input for each file... if len(fileContents) == 1: label = self.captionTitle else: label = None for index in range(len(fileContents)): myInput = Input(fileContents[index], label) segment = myInput[0] segment.annotations.update(annotations[index]) myInput[0] = segment self.createdInputs.append(myInput) # If there's only one file, the widget's output is the created Input. if len(fileContents) == 1: self.segmentation = self.createdInputs[0] # Otherwise the widget's output is a concatenation... else: self.segmentation = Segmenter.concatenate( segmentations=self.createdInputs, label=self.captionTitle, copy_annotations=True, import_labels_as=None, sort=False, auto_number_as=None, merge_duplicates=False, progress_callback=None, ) message = u'%i segment@p sent to output ' % len(self.segmentation) message = pluralize(message, len(self.segmentation)) numChars = 0 for segment in self.segmentation: segmentLength = len(Segmentation.get_data(segment.str_index)) numChars += segmentLength message += u'(%i character@p).' % numChars message = pluralize(message, numChars) self.infoBox.setText(message) progressBar.finish() self.controlArea.setDisabled(False) self.send('Text data', self.segmentation, self) self.sendButton.resetSettingsChangedFlag()
def sendData(self): """Compute result of widget processing and send to output""" # Check that there's an input... if self.inputSeg is None: self.infoBox.setText("Widget needs input", "warning") self.send("Linguistically analyzed data", None, self) return # Initialize progress bar. self.infoBox.setText( u"Processing, please wait...", "warning", ) self.controlArea.setDisabled(True) progressBar = ProgressBar(self, iterations=len(self.inputSeg)) tokenizedSegments = list() # Process each input segment... for segment in self.inputSeg: # Input segment attributes... inputContent = segment.get_content() inputAnnotations = segment.annotations inputString = segment.str_index inputStart = segment.start or 0 inputEnd = segment.end or len(inputContent) # NLP analysis... doc = self.nlp(inputContent) # Process each token in input segment... for token in doc: tokenAnnotations = inputAnnotations.copy() tokenAnnotations.update({ k: getattr(token, k) for k in RELEVANT_KEYS if getattr(token, k) is not None }) tokenStart = inputStart + token.idx tokenizedSegments.append( Segment( str_index=inputString, start=tokenStart, end=tokenStart + len(token), annotations=tokenAnnotations, )) progressBar.advance() outputSeg = Segmentation(tokenizedSegments, self.captionTitle) # Set status to OK and report data size... message = "%i segment@p sent to output." % len(outputSeg) message = pluralize(message, len(outputSeg)) self.infoBox.setText(message) print(outputSeg.to_string()) # Clear progress bar. progressBar.finish() self.controlArea.setDisabled(False) # Send data to output... self.send("Linguistically analyzed data", outputSeg, self) self.sendButton.resetSettingsChangedFlag()