示例#1
0
    def downloadModels(self):
        """Respond to Download button (Model manager tab)."""
        global INSTALLED_MODELS

        # Ask for confirmation...
        num_models = len(self.selectedModels)
        message = "Your are about to download %i language model@p. " +   \
                  "This may take up to several minutes depending on your " +  \
                  "internet connection. Do you want to proceed?"
        message = message % num_models
        buttonReply = QMessageBox.question(self, "Textable",
                                           pluralize(message, num_models),
                                           QMessageBox.Ok | QMessageBox.Cancel)
        if buttonReply == QMessageBox.Cancel:
            return

        # Download models...
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=num_models)
        for model_idx in reversed(self.selectedModels):
            model = self.downloadableModelLabels[model_idx]
            download_spacy_model(AVAILABLE_MODELS[model])
            del self.downloadableModelLabels[model_idx]
            progressBar.advance()

        # Update GUI...
        self.downloadableModelLabels = self.downloadableModelLabels
        self.selectedModels = list()
        progressBar.finish()
        self.controlArea.setDisabled(False)
        message = "Downloaded %i language model@p, please restart " +   \
                  "Orange for changes to take effect."
        message = message % num_models
        QMessageBox.information(None, "Textable",
                                pluralize(message, num_models), QMessageBox.Ok)
    def create_content_segment(self, post, includeImage=False):
        """ Creation of segments for posts"""
        progressBar = ProgressBar(self, iterations=1)
        self.infoBox.setText("Processing post content, please wait.",
                             "warning")
        # Create annotations
        annotations = dict()
        annotations["Title"] = post.title
        annotations["Id"] = post.id
        annotations["Parent"] = post.id
        annotations["Author"] = post.author
        annotations["Score"] = post.score
        annotations["Parent_type"] = "0"

        # Time annotations
        time = post.created_utc
        ts = int(time)
        date = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
        annotations["Posted_Unix"] = time
        annotations["Posted_at"] = date

        # author, created_utc (ou created ?) et score
        content = post.selftext
        if content == "":
            content = "[image]"

        if not (includeImage == False and content == "[image]"):
            # Create a segment is the user wishes to have a segment
            # When there is only an image
            progressBar.advance()
            self.listeTempPosts.append(content)
            self.listeTempAnnot.append(annotations)
            progressBar.finish()

        return
    def get_all_titles(self):
        '''php_query_string and http_query_string are the variable that will need to be changed
        if different database is used or if current database's structure undergoes changes'''
        php_query_string = '/movie_script.php?movie='
        http_query_string = 'https://www.springfieldspringfield.co.uk/' +   \
                            'movie_scripts.php?order='
        alphabet = ['0', 'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 
                    'K', 'L', 'M', 'N', 'O', 'P', 'K', 'R', 'S', 'T', 'U', 
                    'V', 'W', 'X', 'Y', 'Z']

        # Initialize progress bar.
        progressBar = ProgressBar(
            self,
            iterations=len(alphabet)
        )
        self.controlArea.setDisabled(True)

        try:
            for lettre in alphabet:
                page_num = 1
                # 1 tick on the progress bar of the widget
                progressBar.advance()

                # This part of code is what gets all the movie titles from each
                # page of the database
                while True:
                    page_url = http_query_string + '%s&page=%i' % (
                        lettre, 
                        page_num,
                    )
                    page = urllib.request.urlopen(page_url)
                    soup = BeautifulSoup(page, 'html.parser')
                    # script_links is a variable that may need to be changed if 
                    # another database is used or current database undergoes 
                    # change
                    script_links = soup.findAll('a', attrs={'class':
                        re.compile("^script-list-item")})
                    if not script_links:
                        break
                    links = dict()
                    for link in soup.findAll(
                        'a', attrs={'class': re.compile("^script-list-item")}
                    ):
                        links[link.text] =  \
                            link.get('href')[len(php_query_string):]
                    self.title_to_href.update(links)
                    page_num += 1

        except:
            self.infoBox.setText(
                "Couldn't download data from springfieldspringfield website.",
                "error"
            )
        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)
        return self.title_to_href
示例#4
0
    def sendData(self):
        """Compute result of widget processing and send to output"""
        
        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input", "warning")
            del self.headerList[:]
            self.headerList = self.headerList
            self.send("CSV Segmentation", None, self)
            return

        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait...", 
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.inputSeg))


        # Treat...
        for segment in self.csvSeg:
            
            pass
                        
            progressBar.advance()

                 
        # Set status to OK and report data size...
        outputSeg = Segmentation(self.csvSeg, label=self.captionTitle)
        if len(self.contentIsNone) == 0 :
            message = "%i segment@p sent to output." % len(outputSeg)
            message = pluralize(message, len(outputSeg))
            self.infoBox.setText(message)
        # message if one or more segments has no content and has been ignored
        elif len(self.contentIsNone) == 1:
            message = "%i segment@p sent to output. (ignored %i segment with \
            no content)" % (len(outputSeg), len(self.contentIsNone))
            message = pluralize(message, len(outputSeg))
            self.infoBox.setText(message)
        else :
            message = "%i segment@p sent to output. (ignored %i segments with \
            no content)" % (len(outputSeg), len(self.contentIsNone))
            message = pluralize(message, len(outputSeg))
            self.infoBox.setText(message)

        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)
        
        # Send data to output...
        self.send("CSV Segmentation", outputSeg, self)
        
        self.sendButton.resetSettingsChangedFlag()             
    def searchFunction(self):
        self.controlArea.setDisabled(True)

        # Search from the springfieldspringfield.co.uk
        query_string = self.newQuery
        testdict = self.title_to_href

        # Reset and clear the visible widget list
        del self.titleLabels[:]
        self.titleLabels = self.titleLabels
        del self.movie_titles[:]
        self.movie_titles = self.movie_titles

        if query_string != "":
            # Initialize progress bar.
            progressBar = ProgressBar(self, iterations=1)

            self.searchResults = process.extractBests(
                query_string, 
                testdict,
                limit=100000,
                score_cutoff=80
            )

            progressBar.finish()

            progressBar = ProgressBar(self, iterations=len(self.searchResults))

            for key, score, val in self.searchResults:
                self.titleLabels.append(val)
                self.movie_titles.append(val)
                self.path_storage[val] = key
                # 1 tick on the progress bar of the widget
                progressBar.advance()

            self.titleLabels = self.titleLabels
            self.clearButton.setDisabled(False)
            self.controlArea.setDisabled(False)

            # Clear progress bar.
            progressBar.finish()

            if self.searchResults:
                self.infoBox.setText("Search complete")
            elif self.searchResults == []:
                self.infoBox.setText("No result please try again", 'warning')

        else:
            self.infoBox.setText(
                "Please, enter a query in a search bar", 
                "warning"
            )
            self.controlArea.setDisabled(False)
示例#6
0
    def sendData(self):
        """Compute result of widget processing and send to output"""

        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input", "warning")
            self.send("Linguistically analyzed data", None, self)
            return

        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait...",
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.inputSeg))

        # Basic NLP analysis for dev purposes...
        analyzedSegments = list()
        for segment in self.inputSeg:
            analyzedString = ""
            doc = self.nlp(segment.get_content())
            for token in doc:
                analyzedString += "%s\t%s\n" % (token.text, token.pos_)
            analyzedSegments.append(Input(analyzedString))
            progressBar.advance()

        outputSeg = LTTL.Segmenter.concatenate(
            analyzedSegments,
            import_labels_as=None,
            label=self.captionTitle,
        )

        # Set status to OK and report data size...
        message = "%i segment@p sent to output." % len(outputSeg)
        message = pluralize(message, len(outputSeg))
        self.infoBox.setText(message)

        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)

        # Send data to output...
        self.send("Linguistically analyzed data", outputSeg, self)

        self.sendButton.resetSettingsChangedFlag()
 def loadModel(self):
     """(Re-)load language model if needed."""
     # Initialize progress bar.
     self.infoBox.setText(
         u"Loading language model, please wait...", 
         "warning",
     )
     self.controlArea.setDisabled(True)
     progressBar = ProgressBar(self, iterations=1)       
     self.nlp = spacy.load(
         #AVAILABLE_MODELS[self.model],
         "en_core_web_sm",
     )
     self.mustLoad = False
     progressBar.advance()
     progressBar.finish()
     self.controlArea.setDisabled(False)
 def loadModel(self):
     """(Re-)load language model if needed."""
     # Initialize progress bar.
     self.infoBox.setText(
         u"Loading language model, please wait...", 
         "warning",
     )
     self.controlArea.setDisabled(True)
     progressBar = ProgressBar(self, iterations=1)       
     disabled, enabled = self.getComponentStatus()
     self.nlp = spacy.load(
         AVAILABLE_MODELS[self.model], 
         disable=disabled,
     )
     self.loadedComponents = enabled
     self.updateReloadNeededLabels()
     self.mustLoad = False
     progressBar.advance()
     progressBar.finish()
     self.controlArea.setDisabled(False)
示例#9
0
    def sendData(self):
        """Compute result of widget processing and send to output"""
        if not self.importedCorpus:
            self.infoBox.setText("Please select a corpus to import.",
                                 "warning")
            self.send("XML data", None, self)
            return
        progressBar = ProgressBar(self, iterations=1)
        response = requests.get(self.importedCorpus)
        myZip = zipfile.ZipFile(io.BytesIO(response.content))
        for file in myZip.infolist():
            print(file.filename, len(myZip.read(file)))
        progressBar.advance()
        progressBar.finish()

        corpus = self.displayedFolderLabels[self.selectedItems[0]]
        self.importedCorpusLabel.setText("Corpus %s correctly imported." %
                                         corpus)
        self.infoBox.setText("All good!")
        self.sendButton.resetSettingsChangedFlag()
    def loadModel(self):
        """(Re-)load language model if needed."""
        # Display warning, disable UI and initialize progress bar...
        self.infoBox.setText(
            u"Loading language model, please wait...", 
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=1)

        # Load model and reset UI.
        self.nlp = spacy.load(spacy_widget.AVAILABLE_MODELS[self.model])
        progressBar.advance()
        progressBar.finish()
        self.controlArea.setDisabled(False)
        
        # Update char list if there's an input...
        if self.inputSeg:
            self.updateCharacterList()
        
        self.sendButton.settingsChanged()
示例#11
0
    def languageChanged(self):
        """Load the appropriate model according to user choice"""

        self.infoBox.setText(
            u"Loading model, please wait...",
            "warning",
        )

        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=1)

        if self.language == "French":
            self.cv = self.loadModelFR()
        elif self.language == "English":
            self.cv = self.loadModelEN()
        elif self.language == "Portuguese":
            self.cv = self.loadModelPT()
        elif self.language == "Dutch":
            self.cv = self.loadModelLN()
        elif self.language == "German":
            self.cv = self.loadModelDE()
        elif self.language == "Greek":
            self.cv = self.loadModelEL()
        elif self.language == "Italian":
            self.cv = self.loadModelIT()
        elif self.language == "Lithuanian":
            self.cv = self.loadModelLT()
        elif self.language == "Norwegian":
            self.cv = self.loadModelNB()
        elif self.language == "Spanish":
            self.cv = self.loadModelES()

        progressBar.advance()
        progressBar.finish()
        self.controlArea.setDisabled(False)

        self.sendButton.settingsChanged()
示例#12
0
    def create_comments_segments(self, post):
        """ Creation of segments for each comment in the post"""
        # Get the totality of the comments under a post
        post.comments.replace_more(limit=None)
        comments = post.comments.list()

        progressBar = ProgressBar(self, iterations=len(comments))
        self.infoBox.setText(
            "Processing comments, this can take a while, please wait.",
            "warning"
        )

        # Creation of a segment for each comment
        for comment in comments:
            annotations = dict()
            annotations["Title"] = post.title
            annotations["Id"] = comment.id
            annotations["Author"] = comment.author
            annotations["Score"] = comment.score

            # Time annotations
            time = comment.created_utc
            ts = int(time)
            date = datetime.utcfromtimestamp(ts).strftime('%Y-%m-%d %H:%M:%S')
            annotations["Posted_Unix"] = time
            annotations["Posted_at"] = date

            # author, created_utc (ou created ?) et score
            parentId= comment.parent_id.split("_")
            annotations["Parent"] = parentId[1]
            annotations["Parent_type"] = parentId[0][1]

            self.listeTempPosts.append(comment.body)
            self.listeTempAnnot.append(annotations)
            progressBar.advance()
        progressBar.finish()
        return
 def updateCharacterList(self):
     """Update character list based on Charnet output."""
     if self.mustLoad:
         self.loadModel()
     self.controlArea.setDisabled(True)
     progressBar = ProgressBar(self, iterations=4)
     string = " ".join(segment.get_content() for segment in self.inputSeg)
     progressBar.advance()
     self.char_df = charnet.extract_spacy_df(string, self.nlp) # TODO progress
     progressBar.advance()
     self.char_df = charnet.unify_tags(self.char_df)
     progressBar.advance()
     self.char_list = charnet.concatenate_parents(self.char_df, min_occ = 1)
     self.characters = [", ".join(char) for char in self.char_list]
     progressBar.advance()
     progressBar.finish()
     self.controlArea.setDisabled(False)
    def updateCharacterList(self):
        """Update character list based on Charnetto output."""
        # Sanity checks...
        if not self.model or not self.inputSeg:
            return
        
        # Init UI...
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=4)
        
        # Get input strings...
        strings = [segment.get_content() for segment in self.inputSeg]
        progressBar.advance()
        
        # Extract character tokens...
        # if self.sourceType == "Plain text":
            # self.char_df = charnetto.extract_spacy_df(strings, self.nlp)
        # elif self.sourceType == "IMSDB-formatted script":
            # self.char_df = charnetto.extract_movie_df(" ".join(strings))
        self.char_df = charnetto.extract_spacy_df(strings, self.nlp)
        
        # TODO deal with \n in names
        progressBar.advance()
        
        # Unify spaCy tags to match those of flair...  
        self.char_df = charnetto.unify_tags(self.char_df)
        progressBar.advance()
        
        # Collapse characters whose name is the prefix of another.
        self.char_list = charnetto.concatenate_parents(self.char_df, min_occ = 1)

        # Build char list and reset UI.
        self.characters = [", ".join(char) for char in self.char_list]
        progressBar.advance()
        progressBar.finish()
        self.controlArea.setDisabled(False)
        
        # Cache character list for resetting if needed.
        self.cachedCaracters = self.characters[:]
    def sendData(self):
        """Send data from website springfieldspringfield"""
        # Skip if title list is empty:
        if self.myBasket == list():
            self.infoBox.setText(
                "Your corpus is empty, please add some movies first",
                "warning"
            )
            self.segmentation = None
            self.send("Movie transcripts", self.segmentation, self)
            return

        # Clear created Inputs.
        self.clearCreatedInputs()

        annotations = list()
        script_list = list()
        annotations_dict = dict()
        self.controlArea.setDisabled(True)

        # Initialize progress bar.
        progressBar = ProgressBar(self, iterations=len(self.myBasket))

        # This part of code is what fetches the actual script
        try:
            for movie in self.myBasket:
                # Each movie that is in the corpus is split into title and year
                # (rsplit makes sure to only split last occurence) which will 
                # become annotations
                b = copy.copy(movie)
                future_annotation = b.rsplit('(', 1)
                movie_title = future_annotation[0]
                movie_year = future_annotation[-1]
                movie_year = movie_year[:-1]
                annotations_dict["Movie Title"] = movie_title
                annotations_dict["Year of release"] = movie_year
                # It is important to make a copy of dictionary, otherwise each 
                # iteration will replace every element of the annotations list
                annotations.append(annotations_dict.copy())
                # link_end and page_url are the two variables that will have to
                # be changed in case scripts need to be taken from elsewhere
                link_end = self.path_storage[movie]
                page_url = "https://www.springfieldspringfield.co.uk/" +   \
                    "movie_script.php?movie=" + link_end
                page = urllib.request.urlopen(page_url)
                soup = BeautifulSoup(page, 'html.parser')

                # This is what grabs the movie script
                script = soup.find("div", {"class":"movie_script"})

                script_list.append(script.text)

                # 1 tick on the progress bar of the widget
                progressBar.advance()

        except:
            self.infoBox.setText(
                "Couldn't download data from SpringfieldSpringfield website.",
                "error"
            )
            self.controlArea.setDisabled(False)
            return

        # Store downloaded script strings in input objects...
        for script in script_list:
            newInput = Input(script, self.captionTitle)
            self.createdInputs.append(newInput)

       # If there's only one play, the widget"s output is the created Input.
        if len(self.createdInputs) == 1:
            self.segmentation = self.createdInputs[0]

        # Otherwise the widget"s output is a concatenation...
        else:
            self.segmentation = Segmenter.concatenate(
                self.createdInputs,
                self.captionTitle,
                import_labels_as=None,
            )

        # Annotate segments...
        for idx, segment in enumerate(self.segmentation):
            segment.annotations.update(annotations[idx])
            self.segmentation[idx] = segment

        # Clear progress bar.
        progressBar.finish()

        self.controlArea.setDisabled(False)

        # Set status to OK and report data size...
        message = "%i segment@p sent to output " % len(self.segmentation)
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += "(%i character@p)." % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)

        self.send("Movie transcripts", self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
示例#16
0
    def sendData(self):
        """Compute result of widget processing and send to output"""
        # Skip if title list is empty:
        if self.myBasket == list():
            self.infoBox.setText(
                "Your corpus is empty, please add some movies first",
                "warning")
            return

        # Clear created Inputs.
        self.clearCreatedInputs()

        self.controlArea.setDisabled(True)

        # Initialize progress bar.
        progressBar = ProgressBar(self, iterations=len(self.myBasket))

        # Connect to imdb and add elements in lists
        list_review = list()
        list_annotation = list()
        annotations = list()
        try:
            for item in self.myBasket:
                movie = self.ia.get_movie_reviews(item['id'])
                movie_annotations = self.ia.get_movie(item['id'])
                list_review.append(movie)
                list_annotation.append(movie_annotations)
                # 1 tick on the progress bar of the widget
                progressBar.advance()

        # If an error occurs (e.g. http error, or memory error)...
        except:
            # Set Info box and widget to "error" state.
            self.infoBox.setText("Couldn't download data from imdb", "error")
            self.controlArea.setDisabled(False)
            return

        # Store movie critics strings in input objects...
        for movie in list_review:
            data = movie.get('data', "")
            reviews_data = data.get('reviews')
            for review in reviews_data:
                reviews = review.get('content')
                newInput = Input(reviews)
                self.createdInputs.append(newInput)
        for item in list_annotation:
            print(item)
            # Store the annotation as dicts in a separate list
            annotations_dict = {"title": item, "year": item["year"]}
            annot_dict_copy = annotations_dict.copy()
            for i in range(25):
                annotations.append(annot_dict_copy)
        print(annotations)
        # If there's only one item, the widget's output is the created Input.
        if len(self.createdInputs) == 1:
            self.segmentation = self.createdInputs[0]

        # Otherwise the widget's output is a concatenation...
        else:
            self.segmentation = Segmenter.concatenate(
                self.createdInputs,
                import_labels_as=None,
            )

        # Annotate segments...
        for idx, segment in enumerate(self.segmentation):
            segment.annotations.update(annotations[idx])
            self.segmentation[idx] = segment

        # Clear progress bar.
        progressBar.finish()

        self.controlArea.setDisabled(False)

        # Set status to OK and report data size...
        message = f"{len(self.segmentation)} segment@p sent to output"
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += " (%i character@p)." % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)

        self.send('Segmentation', self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
示例#17
0
    def searchMovies(self):
        """Search from imdb movie database"""
        result_list = {}
        query_string = self.newQuery

        if query_string != "":
            counter = 1
            counter_max = int(self.nbr_results)
            result_id = 0

            self.controlArea.setDisabled(True)

            # Initialize progress bar
            progressBar = ProgressBar(self, iterations=counter_max)
            filtered_results = list()
            if self.type_results == 'Title':

                # movie name
                movie_name = query_string

                # searching the movie
                search = self.ia.search_movie(movie_name)
                for film in search:
                    if 'year' in film:
                        filtered_results.append(film)

            elif self.type_results == 'Actor':
                actor_name = query_string
                people = self.ia.search_person(actor_name)
                searched_actor = people[0].personID
                first_search = self.ia.get_person_filmography(searched_actor)

                # Checks if the user input is a valid actor/actress
                if 'actor' in first_search['data']['filmography']:
                    search = first_search['data']['filmography']['actor']
                elif 'actress' in first_search['data']['filmography']:
                    search = first_search['data']['filmography']['actress']
                else:
                    search = list()
                    self.infoBox.setText(
                        "Please enter a valid actor or actress name",
                        "warning")

                # Checks if the movie has a year associated and stores it in a list
                filtered_results = [film for film in search if 'year' in film]

            if self.filter_results == 'Random':
                random.shuffle(filtered_results)

            elif self.filter_results == 'Alphabetical':
                alpha_dict = dict()
                for result in filtered_results:
                    my_id = result.movieID
                    alpha_dict[str(result)] = my_id
                    print(alpha_dict)
                sorted_dict = sorted(alpha_dict.keys(),
                                     key=lambda x: x.lower())
                print(sorted_dict)
                filtered_results = list()
                for i in sorted_dict:
                    value = alpha_dict[i]
                    print(value)
                    print(self.ia.get_movie(value))
                    filtered_results.append(self.ia.get_movie(value))

            # Each result is stored in a dictionnary with its title
            # and year of publication if it is specified

            for result in filtered_results:
                if counter <= counter_max:
                    try:
                        result_id += 1
                        year = result['year']
                        movie_id = result.movieID
                        result_list[result_id] = {
                            'name': result,
                            'year': year,
                            'id': movie_id
                        }
                    except KeyError:
                        continue
                    counter += 1
                else:
                    break

            # 1 tick on the progress bar of the widget
            progressBar.advance()
            # Stored the results list in the "result_list" variable
            self.searchResults = result_list

            # Reset and clear the visible widget list
            del self.titleLabels[:]

            # Update the results list with the search results
            # in order to display them
            for idx in self.searchResults:
                result_string = f'{self.searchResults[idx]["name"]} - {self.searchResults[idx]["year"]}'
                self.titleLabels.append(result_string)

            self.titleLabels = self.titleLabels
            self.clearButton.setDisabled(False)
            self.addButton.setDisabled(self.selectedTitles == list())

            # Clear progress bar.
            progressBar.finish()
            self.controlArea.setDisabled(False)

        else:
            self.infoBox.setText("Please type something in the search bar",
                                 "warning")
    def sendData(self):
        """Compute result of widget processing and send to output."""

        # Check that there's a model...
        if not self.model:
            self.infoBox.setText(
                "Please download a language model first.",
                "warning",
            )
            self.tabs.setCurrentIndex(1)
            return

        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input.", "warning")
            for channel in [c.name for c in self.outputs]:
                self.send(channel, None, self)
            return

        # Check max length and adjust if needed...
        inputLength = sum(len(s.get_content()) for s in self.inputSeg)
        if self.maxLen != "no limit":
            maxNumChar = int(self.maxLen.split()[0]) * 1000000
            if inputLength > maxNumChar:
                self.infoBox.setText(
                    "Input exceeds max number of characters set by user.", 
                    "warning",
                )
                for channel in [c.name for c in self.outputs]:
                    self.send(channel, None, self)
                return
        else:
            if inputLength > self.nlp.max_length:
                maxNumChar = inputLength          
        
        # Load components if needed...
        disabled, enabled = self.getComponentStatus()
        if self.mustLoad or not(
            self.nlp and set(enabled) <= set(self.loadedComponents)
        ):
            self.loadModel()
        self.nlp.max_length = maxNumChar
        
        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait...", 
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.inputSeg))       

        tokenSegments = list()
        entitySegments = list()
        chunkSegments = list()
        sentenceSegments = list()
        
        # Process each input segment...
        for segment in self.inputSeg:
        
            # NLP analysis...
            disabled, _ = self.getComponentStatus()
            disabled = [c for c in disabled if c in set(self.loadedComponents)]
            with self.nlp.disable_pipes(*disabled):
                doc = self.nlp(segment.get_content())

            # Get token segments...
            tokenSegments.extend(spacyItemsToSegments(doc, segment))

            # Get named entity segments...
            if self.segmentEntities:
                entitySegments.extend(spacyItemsToSegments(doc.ents, segment))

            # Get noun chunk segments...
            if self.segmentChunks:
                chunkSegments.extend(
                    spacyItemsToSegments(doc.noun_chunks, segment), 
                )

            # Get sentences segments...
            if self.segmentSentences:
                sentenceSegments.extend(
                    spacyItemsToSegments(doc.sents, segment), 
                )

            progressBar.advance()

        # Build segmentations and send them to output...                   
        tokenSeg = Segmentation(tokenSegments, self.captionTitle + "_tokens")
        self.send("Tokenized text", tokenSeg, self)
        if self.segmentChunks:
            chunkSeg = Segmentation(
                chunkSegments, 
                self.captionTitle + "_chunks",
            )
            self.send("Noun chunks", chunkSeg, self)
        if self.segmentEntities:
            entitySeg = Segmentation(
                entitySegments, 
                self.captionTitle + "_entities",
            )
            self.send("Named entities", entitySeg, self)
        if self.segmentSentences:
            sentenceSeg = Segmentation(
                sentenceSegments, 
                self.captionTitle + "_sentences",
            )
            self.send("Sentences", sentenceSeg, self)

        # Set status to OK and report data size...
        message = "%i token@p" % len(tokenSeg)
        message = pluralize(message, len(tokenSeg))
        if self.segmentChunks:
            message += ", %i chunk@p" % len(chunkSeg)
            message = pluralize(message, len(chunkSeg))
        if self.segmentEntities:
            message += ", %i " % len(entitySeg)
            message += "entity" if len(entitySeg) == 1 else "entities"
        if self.segmentSentences:
            message += ", %i sentence@p" % len(sentenceSeg)
            message = pluralize(message, len(sentenceSeg))
        message += " sent to output."
        last_comma_idx = message.rfind(",")
        if last_comma_idx > -1:
            message = message[:last_comma_idx] + " and" +    \
                message[last_comma_idx+1:]
        self.infoBox.setText(message)
        
        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)
                
        self.sendButton.resetSettingsChangedFlag()             
示例#19
0
    def treat_input(self):
        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input", "warning")
            del self.headerList[:]
            self.headerList = self.headerList
            return

        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait...", 
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.inputSeg))

        # clear lists
        del self.csvSeg[:]
        del self.contentIsNone[:]

        # Process each input segment...
        for segment in self.inputSeg:
        
            # Input segment attributes...
            inputContent = segment.get_content()
            if not self.deleteQuotes == False :
                inputContent = inputContent.replace('"',"")
            inputAnnotations = segment.annotations
            inputStrIdx = segment.str_index
            inputStart = segment.start or 0
            inputEnd = segment.end or len(inputContent)
            #Call data processing
            csv_stream = io.StringIO(inputContent)
            dialect = sniffer.sniff(csv_stream.readline())
            dialect.quoting=csv.QUOTE_NONE
            csv_stream.seek(0)
            my_reader = csv.reader(csv_stream, dialect)
            position = 0
            # Process each seg in inputContent
            for seg in inputContent:
                segAnnotations = inputAnnotations.copy()
            # This  will launch if sniffer detects a header in the content.
            if sniffer.has_header(inputContent) == True:
                # go back to the start otherwise we're going to start from the
                # second row
                csv_stream.seek(0)
                # the header row is defined here.
                if self.isRenamed == False :
                    self.dict_keys = next(my_reader)
                    for key in self.dict_keys:
                    # this is position of first content
                    # TODO : separator length (if not 1)
                        position += (len(key) + 1)
                else :
                    input_keys = next(my_reader)
                    for key in input_keys:
                    # this is position of first content
                    # TODO : separator length (if not 1)
                        position += (len(key) + 1)


            # This will launch if sniffer does not detect a header 
            # in the content.
            if sniffer.has_header(inputContent) == False:
                # go back to the start otherwise we're going to start from the
                # second row. we do this here even though we don't really care
                # about the first row simply because in general we consider the
                # first row to not have any missing values
                csv_stream.seek(0)
                first_row = next(my_reader)
                n_cols = len(first_row)
                if self.isRenamed == False :
                    self.dict_keys = list()
                    for item in range(1, n_cols+1):
                        self.dict_keys.append(str(item))
                csv_stream.seek(0)


            # clear the list before appending
            del self.headerList[:]

            for key in self.dict_keys:
                # appends the headers to the gui list
                if self.dict_keys.index(key) == self.content_column:
                    self.headerList.append(str(key)+"(*content)")
                    self.headerList = self.headerList
                else :
                    self.headerList.append(str(key))
                    self.headerList = self.headerList


            for idx, row in enumerate(my_reader, start=2):
                # Get old annotations in new dictionary
                oldAnnotations = inputAnnotations.copy()
                segAnnotations = dict()
                # initiate next row starting position
                next_position = position
                for key in oldAnnotations.keys():
                    segAnnotations[key] = oldAnnotations[key]

                # This is the main part where we transform our data into
                # annotations.
                for key in self.dict_keys:
                    # segAnnotations["length"] = position
                    # segAnnotations["row"] = str(row)

                    # if column is content (first column (0) by default)
                    if self.dict_keys.index(key) == self.content_column:
                        # put value as content
                        content = row[self.dict_keys.index(key)]
                    # else we put value in annotation
                    else:
                        # only if value is not None
                        if len(row[self.dict_keys.index(key)]) != 0 :
                            segAnnotations[key] = row[self.dict_keys.index(key)]
                    # implement position and next_position depending on
                    # content column
                    if self.dict_keys.index(key) < self.content_column:
                        position += len(row[self.dict_keys.index(key)]) + 1
                        next_position += len(row[self.dict_keys.index(key)]) + 1
                    if self.dict_keys.index(key) >= self.content_column:
                        next_position += len(row[self.dict_keys.index(key)]) + 1

                if len(content) != 0:
                    self.csvSeg.append(
                        Segment(
                            str_index = inputStrIdx,
                            start = position,
                            end = position + len(content),
                            annotations = segAnnotations
                            )
                        )

                else :
                    # if no content, add idx of the row and do not append
                    # TODO : something with contentIsNone
                    self.contentIsNone.append(idx)

                # initiate new row starting position
                position = next_position
                        
            progressBar.advance()

        unSeg = len(self.csvSeg)         
        # Set status to OK and report segment analyzed...
        message = "%i segment@p analyzed." % unSeg
        message = pluralize(message, unSeg)
        message += " (Ignored %i segment@p with no content)" %      \
            len(self.contentIsNone)
        message = pluralize(message, len(self.contentIsNone))
        self.infoBox.setText(message)

        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)

        self.sendButton.resetSettingsChangedFlag()
        self.sendButton.sendIf()
    def sendData(self):
        """Compute result of widget processing and send to output"""

        # Clear morphology...
        self.morphology = dict()

        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input", "warning")
            self.send("Morphologically analyzed data", None, self)
            self.updateGUI()
            return

        # Perform morphological analysis...

        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait (word count)...",
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=100)

        # Word count...
        wordCounts = collections.Counter(
            [segment.get_content() for segment in self.inputSeg])
        self.morphology["wordCounts"] = wordCounts
        self.infoBox.setText(
            u"Processing, please wait (signature extraction)...",
            "warning",
        )
        progressBar.advance(5)  # 5 ticks on the progress bar...

        # Learn signatures...
        try:
            lxa5crab.crab_nebula.MIN_STEM_LEN = self.minStemLen
            signatures, stems, suffixes = lxa5crab.find_signatures(wordCounts)
            self.morphology["signatures"] = signatures
            self.morphology["stems"] = stems
            self.morphology["suffixes"] = suffixes
        except ValueError as e:
            self.infoBox.setText(e.__str__(), "warning")
            self.send("Morphologically analyzed data", None, self)
            self.controlArea.setDisabled(False)
            progressBar.finish()  # Clear progress bar.
            self.morphology = dict()
            self.updateGUI()
            return
        self.infoBox.setText(
            u"Processing, please wait (word parsing)...",
            "warning",
        )
        progressBar.advance(80)

        # Parse words...
        parser = lxa5crab.build_parser(wordCounts, signatures, stems, suffixes)
        self.morphology["parser"] = parser
        newSegments = list()
        num_analyzed_words = 0
        for segment in self.inputSeg:
            parses = parser[segment.get_content()]
            newSegment = segment.deepcopy()
            if parses[0].signature:
                num_analyzed_words += 1
            newSegment.annotations.update(
                {
                    "stem": parses[0].stem,
                    "suffix": parses[0].suffix  \
                                if len(parses[0].suffix) else "NULL",
                    "signature": parses[0].signature
                }
            )
            newSegments.append(newSegment)
        self.send(
            "Morphologically analyzed data",
            Segmentation(newSegments, self.captionTitle),
            self,
        )
        self.updateGUI()
        progressBar.advance(15)

        # Set status to OK and report data size...
        message = "%i segment@p sent to output (%.2f%% analyzed)." % (len(
            self.inputSeg), (num_analyzed_words / len(self.inputSeg) * 100))
        message = pluralize(message, len(self.inputSeg))
        self.infoBox.setText(message)

        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)

        self.sendButton.resetSettingsChangedFlag()
示例#21
0
class Treetagger(OWTextableBaseWidget):
    """Orange widget for POS-tagging and lemmatization with Treetagger"""

    name = "Treetagger"
    description = "POS-tagging and lemmatization with Treetagger"
    icon = "icons/treetagger.svg"
    priority = 2003

    inputs = [("Segmentation", Segmentation, "inputData")]
    outputs = [("Tagged data", Segmentation)]

    settingsHandler = VersionedSettingsHandler(
        version=__version__.rsplit(".", 1)[0])

    language = settings.Setting(0)
    replaceUnknown = settings.Setting(False)
    outputFormat = settings.Setting("segment into words")

    want_main_area = False

    configFilePath = os.path.normpath(
        appdirs.user_data_dir("textable", "langtech") + "/treetagger_path")

    def __init__(self, *args, **kwargs):
        """Initialize a Message widget"""
        super().__init__(*args, **kwargs)

        # Other attributes...
        self.segmentation = None
        self.createdInputs = list()
        self.noLanguageParameterWarning = (
            "Please make sure that at least one language parameter "
            "file is installed in your Treetagger 'lib' directory, "
            "then click 'Reload language parameter files'.")
        self.noTreetaggerPathWarning = (
            "Please click 'Locate Treetagger' below and select the "
            "base directory of a valid Treetagger distribution.")
        self.TreetaggerPath = (treetaggerwrapper.locate_treetagger()
                               or self.lookupSavedTreetaggerPath())

        self.infoBox = InfoBox(widget=self.controlArea)

        self.sendButton = SendButton(widget=self.controlArea,
                                     master=self,
                                     callback=self.sendData,
                                     infoBoxAttribute=u"infoBox",
                                     sendIfPreCallback=self.updateGUI)

        gui.separator(self.controlArea, height=3)

        self.optionsBox = gui.widgetBox(
            self.controlArea,
            u"Options",
        )

        self.languageCombobox = gui.comboBox(
            widget=self.optionsBox,
            master=self,
            value="language",
            items=list(),
            sendSelectedValue=True,
            orientation=u"horizontal",
            label="Input language:",
            labelWidth=180,
            callback=self.sendButton.settingsChanged,
            tooltip=(u"Select the language of the input text."),
        )
        self.languageCombobox.setMinimumWidth(120)

        gui.separator(self.optionsBox, height=3)

        gui.comboBox(
            widget=self.optionsBox,
            master=self,
            value="outputFormat",
            items=[
                "segment into words",
                "add XML tags",
            ],
            sendSelectedValue=True,
            orientation=u"horizontal",
            label="Output format:",
            labelWidth=180,
            callback=self.sendButton.settingsChanged,
            tooltip=(
                u"Select the format of the output:\n\n"
                u"Segment into words: each word is in a separate segment,\n"
                u"with lemma and POS-tag as annotations.\n\n"
                u"Add XML tags: output segments correspond to input segments\n"
                u"and each word is tagged in XML as a 'w' element with\n"
                u"lemma and POS-tag as attributes."),
        )

        gui.separator(self.optionsBox, height=3)

        gui.checkBox(
            widget=self.optionsBox,
            master=self,
            value="replaceUnknown",
            label="Output token in place of [unknown] lemmas",
            callback=self.sendButton.settingsChanged,
            tooltip=(
                u"For out-of-vocabulary words, the word form is used as the\n"
                u"lemma (in place of Treetagger's default 'unknown' code)."),
        )

        gui.rubber(self.controlArea)

        self.sendButton.draw()
        self.infoBox.draw()

        self.locateTreetaggerBox = gui.widgetBox(
            self.controlArea,
            addSpace=False,
        )

        gui.separator(self.locateTreetaggerBox, height=3)

        self.treetaggerButton = gui.button(
            widget=self.locateTreetaggerBox,
            master=self,
            label="Locate Treetagger",
            callback=self.validateTreetagger,
            tooltip=(
                u"Click to select the location of the Treetagger base\n"
                u"directory (containing the 'lib' and 'bin' subdirectories)."),
        )

        self.sendButton.sendIf()

        self.adjustSizeWithTimer()

    def inputData(self, inputData):
        """Process incoming data."""
        self.segmentation = inputData
        self.infoBox.inputChanged()
        self.sendButton.sendIf()

    def sendData(self):

        # Clear created Inputs...
        self.clearCreatedInputs()

        if not self.TreetaggerPath:
            self.infoBox.setText(self.noTreetaggerPathWarning, "warning")
            self.send("Tagged data", None)
            return
        elif not self.getAvailableLanguages():
            self.infoBox.setText(self.noLanguageParameterWarning, "warning")
            self.send("Tagged data", None)
            return
        elif not self.segmentation:
            self.infoBox.setText(u"Widget needs input", "warning")
            self.send("Tagged data", None)
            return

        # Initialize progress bar.
        self.infoBox.setText(u"Processing, please wait...", "warning")
        self.controlArea.setDisabled(True)
        self.progressBar = ProgressBar(self, iterations=5)

        # Create a copy of input seg, storing annotations in temp attr...
        copy_of_input_seg = Segmentation()
        copy_of_input_seg.label = self.segmentation.label
        for seg_idx, segment in enumerate(self.segmentation):
            attr = " ".join([
                "%s=%s" % (
                    ''.join(c for c in unicodedata.normalize('NFD', item[0])
                            if unicodedata.category(c) != 'Mn'),
                    quoteattr(str(item[1])),
                ) for item in segment.annotations.items()
            ])

            segment.annotations["tt_ax"] = attr
            copy_of_input_seg.append(segment)

        self.progressBar.advance()

        # Dump segmentation in unique string to avoid multiple calls to TT...
        concatenated_text = copy_of_input_seg.to_string(
            formatting="<ax_tt %(tt_ax)s>%(__content__)s</ax_tt>",
            display_all=True,
        )

        self.progressBar.advance()

        # Tag the segmentation contents...
        tagopt = '-token -lemma -sgml -quiet'
        if self.replaceUnknown:
            tagopt += " -no-unknown"
        tagger = treetaggerwrapper.TreeTagger(
            TAGLANG=pycountry.languages.get(name=self.language).alpha_2,
            TAGOPT=tagopt,
            TAGDIR=self.TreetaggerPath,
        )
        tagged_lines = tagger.tag_text(
            concatenated_text,
            notagurl=True,
            notagemail=True,
            notagip=True,
            notagdns=True,
        )
        tagged_input = Input("\n".join(tagged_lines))
        self.createdInputs.append(tagged_input)

        # Replace <unknown> with [unknown] and " with &quot; then
        # re-segment to match the original segmentation structure.
        tagged_segmentation, _ = Segmenter.recode(
            tagged_input,
            substitutions=[
                (re.compile(r"<unknown>"), "[unknown]"),
                (re.compile(r'"""'), '"&quot;"'),
            ],
        )
        tagged_segmentation = Segmenter.import_xml(tagged_segmentation,
                                                   "ax_tt")

        self.progressBar.advance()

        # Place each output line of Treetagger in an xml tag with annotations..
        xml_segmentation, _ = Segmenter.recode(
            tagged_segmentation,
            substitutions=[
                (re.compile(r"(.+)\t(.+)\t(.+?)(?=[\r\n])"),
                 '<w lemma="&3" pos-tag="&2">&1</w>'),
                (re.compile(r'^\n|\n$'), ''),
            ],
        )
        # Segment into individual tokens if XML output option is disabled...
        if self.outputFormat == "add XML tags":
            output_segmentation = xml_segmentation
        else:
            try:
                output_segmentation = Segmenter.import_xml(
                    xml_segmentation, "w")
            except ValueError:
                self.infoBox.setText(
                    "Please check that either the input contains well-formed "
                    "XML, or it doesn't contain instances of '&#60;' and '\x3e'",
                    "error")
                self.send("Tagged data", None)
                self.progressBar.finish()
                self.controlArea.setDisabled(False)
                return

        self.progressBar.finish()
        self.controlArea.setDisabled(False)

        output_segmentation.label = self.captionTitle
        message = u'%i segment@p sent to output.' % len(output_segmentation)
        message = pluralize(message, len(output_segmentation))
        self.infoBox.setText(message)
        self.send('Tagged data', output_segmentation, self)
        self.sendButton.resetSettingsChangedFlag()

    def updateGUI(self):
        """Update GUI state"""
        if self.TreetaggerPath:
            self.optionsBox.setDisabled(False)
            self.locateTreetaggerBox.setVisible(False)
            self.languageCombobox.clear()
            languages = self.getAvailableLanguages()
            if not languages:
                self.infoBox.setText(self.noLanguageParameterWarning,
                                     "warning")
                self.optionsBox.setDisabled(True)
                self.locateTreetaggerBox.setVisible(True)
                self.treetaggerButton.setText(
                    "Reload language parameter files")
            else:
                self.language = self.language or languages[0]
        else:
            self.infoBox.setText(self.noTreetaggerPathWarning, "warning")
            self.optionsBox.setDisabled(True)
            self.locateTreetaggerBox.setVisible(True)
        self.adjustSizeWithTimer()

    def getAvailableLanguages(self):
        languages = list()
        for lang_code in sorted(treetaggerwrapper.g_langsupport):
            if lang_code.startswith("__"):
                continue
            try:
                treetaggerwrapper.TreeTagger(
                    TAGLANG=lang_code,
                    TAGDIR=self.TreetaggerPath,
                )
                language = pycountry.languages.get(alpha_2=lang_code).name
                self.languageCombobox.addItem(language)
                languages.append(language)
            except:
                pass
        return languages

    def lookupSavedTreetaggerPath(self):
        """Look for a saved Treetagger base dir path in app data"""
        if os.path.exists(self.__class__.configFilePath):
            try:
                inputFile = open(self.__class__.configFilePath, "r")
                TreetaggerSavedPath = inputFile.read()
                inputFile.close()
                if self.checkTreetaggerPath(TreetaggerSavedPath):
                    return TreetaggerSavedPath
                else:
                    os.remove(self.__class__.configFilePath)
                    return None
            except IOError:
                pass

    def validateTreetagger(self):
        """Respond to user actions needed to validate Treetagger path"""

        # If the Treetagger path is known, make sure there are language files...
        if self.TreetaggerPath:
            if self.getAvailableLanguages():
                self.sendButton.settingsChanged()
                self.updateGUI()
            else:
                QMessageBox.warning(None, 'Textable',
                                    'Language parameter files not found.',
                                    QMessageBox.Ok)
            return

        # Else if the path is not known...

        # First try to locate it automatically...
        TreetaggerPath = treetaggerwrapper.locate_treetagger()

        # If it fails, let the user locate it manually...
        if not (TreetaggerPath and self.checkTreetaggerPath(TreetaggerPath)):

            TreetaggerManualPath = os.path.normpath(
                str(
                    QFileDialog.getExistingDirectory(
                        self, u"Please locate Treetagger base directory")))

            # If user selected a dir...
            if TreetaggerManualPath:

                # Check if selected dir contains Treetagger binary...
                if self.checkTreetaggerPath(TreetaggerManualPath):
                    TreetaggerPath = TreetaggerManualPath
                else:
                    QMessageBox.warning(
                        None, 'Textable',
                        'Not a valid Treetagger base directory.',
                        QMessageBox.Ok)

        # If a valid path was found somehow, save config to app data...
        if TreetaggerPath:
            try:
                user_data_editor_dir = os.path.normpath(
                    self.__class__.configFilePath + "/../..")
                if not os.path.exists(user_data_editor_dir):
                    os.makedirs(user_data_editor_dir)
                user_data_software_dir = os.path.normpath(
                    self.__class__.configFilePath + "/..")
                if not os.path.exists(user_data_software_dir):
                    os.makedirs(user_data_software_dir)
                outputFile = open(self.__class__.configFilePath, "w")
                outputFile.write(TreetaggerPath)
                outputFile.close()
            except IOError:
                pass
            self.TreetaggerPath = TreetaggerPath

            self.sendButton.settingsChanged()

    def checkTreetaggerPath(self, path):
        """Check if path is a valid Treetagger base dir"""
        return os.path.exists(
            os.path.normpath(path + "/bin/tree-tagger" +
                             (".exe" if os.name == "nt" else "")))

    def clearCreatedInputs(self):
        for i in self.createdInputs:
            Segmentation.set_data(i[0].str_index, None)
        del self.createdInputs[:]

    def onDeleteWidget(self):
        """Free memory when widget is deleted (overriden method)"""
        self.clearCreatedInputs()

    def setCaption(self, title):
        if 'captionTitle' in dir(self):
            changed = title != self.captionTitle
            super().setCaption(title)
            if changed:
                self.sendButton.settingsChanged()
        else:
            super().setCaption(title)
    def searchMovies(self):
        """Search from imdb movie database"""
        result_list = {}
        query_string = self.newQuery

        if query_string != "":
            counter = 1
            counter_max = int(self.nbr_results)
            result_id = 0
            result_artist = []

            self.controlArea.setDisabled(True)

            # Initialize progress bar
            progressBar = ProgressBar(self, iterations=counter_max)

            ia = imdb.IMDb()

            # movie name
            name = query_string

            # searching the movie
            search = ia.search_movie(name)
            print(search)

            # Each result is stored in a dictionnary with its title
            # and year of publication if it is specified
            for result in search:
                if counter <= counter_max:
                    #print(counter)
                    #print(counter_max)
                    try:
                        result_id += 1
                        year = result['year']
                        movie_id = result.movieID
                        result_list[result_id] = {
                            'name': result,
                            'year': year,
                            'id': movie_id
                        }
                    except KeyError:
                        result_id += 1
                        result_list[result_id] = {
                            'name': result,
                        }

                    counter += 1
                else:
                    break

            # 1 tick on the progress bar of the widget
            progressBar.advance()
            # Stored the results list in the "result_list" variable
            self.searchResults = result_list

            # Reset and clear the visible widget list
            del self.titleLabels[:]

            # Update the results list with the search results
            # in order to display them
            for idx in self.searchResults:
                try:
                    result_string = f'{self.searchResults[idx]["name"]} - {self.searchResults[idx]["year"]}'
                    self.titleLabels.append(result_string)
                except KeyError:
                    result_string = f'{self.searchResults[idx]["name"]}'
                    self.titleLabels.append(result_string)

            self.titleLabels = self.titleLabels
            self.clearButton.setDisabled(False)
            self.addButton.setDisabled(False)

            # Clear progress bar.
            progressBar.finish()
            self.controlArea.setDisabled(False)

        else:
            self.infoBox.setText("Please enter a movie title", "warning")
示例#23
0
    def sendData(self):
        """Compute result of widget processing and send to output"""
        # Skip if title list is empty:
        if self.myBasket == list():
            self.infoBox.setText(
                "Your corpus is empty, please add some books first", "warning")
            return

        # Clear created Inputs.
        self.clearCreatedInputs()

        self.controlArea.setDisabled(True)

        # Initialize progress bar.
        progressBar = ProgressBar(
            self,
            iterations=len(self.myBasket),
        )

        selectedTexts = list()
        text_content = list()
        annotations = list()
        # get the Gutenberg cache
        cache = GutenbergCache.get_cache()
        try:
            # TODO: Retrieve selected texts from gutenberg
            for text in self.myBasket:

                # Get the id of the text
                query_id = cache.native_query(
                    sql_query=
                    "select gutenbergbookid from books where id == {selected_id}"
                    .format(selected_id=text[2]))
                gutenberg_id = list(query_id)

                # Get the text with Gutenbergpy
                gutenberg_text = gutenbergpy.textget.strip_headers(
                    gutenbergpy.textget.get_text_by_id(gutenberg_id[0][0]))
                text_content.append(gutenberg_text)

                annotations.append(text[1])
                progressBar.advance()

        # If an error occurs (e.g. http error, or memory error)...
        except Exception:
            # Set Info box and widget to "error" state.
            self.infoBox.setText("Couldn't download data from Gutenberg",
                                 "error")
            self.controlArea.setDisabled(False)
            return

        # TODO: send gutenberg texts as output
        # Store downloaded lyrics strings in input objects...
        for text in text_content:
            newInput = Input(text, self.captionTitle)
            self.createdInputs.append(newInput)

        # If there"s only one play, the widget"s output is the created Input.
        if len(self.createdInputs) == 1:
            self.segmentation = self.createdInputs[0]

        # Otherwise the widget"s output is a concatenation...
        else:
            self.segmentation = Segmenter.concatenate(
                self.createdInputs,
                self.captionTitle,
                import_labels_as=None,
            )

        # TODO: annotate with book metadata
        # Annotate segments...
        for idx, segment in enumerate(self.segmentation):
            segment.annotations.update({"title": annotations[idx]})
            self.segmentation[idx] = segment

        # Clear progress bar.
        progressBar.finish()

        self.controlArea.setDisabled(False)

        # Set status to OK and report data size...
        message = "%i segment@p sent to output " % len(self.segmentation)
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += "(%i character@p)." % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)

        self.send("Gutenberg importation", self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
    def searchFunction(self):
        """Search from website Genius"""

        result_list = {}
        query_string = self.newQuery

        if query_string != "":
            page = 1
            page_max = int(self.nbr_results) / 10
            result_id = 0
            result_artist = []

            self.controlArea.setDisabled(True)

            # Initialize progress bar.
            progressBar = ProgressBar(self, iterations=page_max)

            while page <= page_max:
                values = {'q': query_string, 'page': page}
                data = urllib.parse.urlencode(values)
                query_url = 'http://api.genius.com/search?' + data
                json_obj = self.url_request(query_url)
                body = json_obj["response"]["hits"]

                # Each result is stored in a dictionnary with its title,
                # artist's name, artist's ID and URL path
                for result in body:
                    result_id += 1
                    title = result["result"]["title"]
                    artist = result["result"]["primary_artist"]["name"]
                    artist_id = result["result"]["primary_artist"]["id"]
                    path = result["result"]["path"]
                    result_list[result_id] = {
                        'artist': artist,
                        'artist_id': artist_id,
                        'path': path,
                        'title': title
                    }
                page += 1

                # 1 tick on the progress bar of the widget
                progressBar.advance()
            # Stored the results list in the "result_list" variable
            self.searchResults = result_list

            # Reset and clear the visible widget list
            del self.titleLabels[:]

            # Update the results list with the search results
            # in order to display them
            for idx in self.searchResults:
                result_string = self.searchResults[idx]["title"] + " - " + \
                                self.searchResults[idx]["artist"]
                self.titleLabels.append(result_string)

            self.titleLabels = self.titleLabels
            self.clearButton.setDisabled(False)
            self.addButton.setDisabled(self.selectedTitles == list())

            # Clear progress bar.
            progressBar.finish()
            self.controlArea.setDisabled(False)

        else:
            self.infoBox.setText("You didn't search anything", "warning")
    def sendData(self):
        """Compute result of widget processing and send to output"""
        # Skip if title list is empty:
        if self.myBasket == list():
            self.infoBox.setText(
                "Your corpus is empty, please add some songs first", "warning")
            return

        # Clear created Inputs.
        self.clearCreatedInputs()

        self.controlArea.setDisabled(True)

        # Initialize progress bar.
        progressBar = ProgressBar(self, iterations=len(self.myBasket))

        # Attempt to connect to Genius and retrieve lyrics...
        selectedSongs = list()
        song_content = list()
        annotations = list()
        try:
            for song in self.myBasket:
                # song is a dict {'idx1':{'title':'song1'...},
                # 'idx2':{'title':'song2'...}}
                page_url = "http://genius.com" + song['path']
                lyrics = self.html_to_text(page_url)
                song_content.append(lyrics)
                annotations.append(song.copy())
                # 1 tick on the progress bar of the widget
                progressBar.advance()

        # If an error occurs (e.g. http error, or memory error)...
        except:
            # Set Info box and widget to "error" state.
            self.infoBox.setText("Couldn't download data from Genius website.",
                                 "error")
            self.controlArea.setDisabled(False)
            return

        # Store downloaded lyrics strings in input objects...
        for song in song_content:
            newInput = Input(song, self.captionTitle)
            self.createdInputs.append(newInput)

        # If there"s only one play, the widget"s output is the created Input.
        if len(self.createdInputs) == 1:
            self.segmentation = self.createdInputs[0]

        # Otherwise the widget"s output is a concatenation...
        else:
            self.segmentation = Segmenter.concatenate(
                self.createdInputs,
                self.captionTitle,
                import_labels_as=None,
            )

        # Annotate segments...
        for idx, segment in enumerate(self.segmentation):
            segment.annotations.update(annotations[idx])
            self.segmentation[idx] = segment

        # Clear progress bar.
        progressBar.finish()

        self.controlArea.setDisabled(False)

        # Set status to OK and report data size...
        message = "%i segment@p sent to output " % len(self.segmentation)
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += "(%i character@p)." % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)

        self.send("Lyrics importation", self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
    def sendData(self):
        """Compute result of widget processing and send to output"""

        # Skip if title list is empty:
        if self.titleLabels == list():
            return

        # Check that something has been selected...
        if len(self.selectedTitles) == 0:
            self.infoBox.setText("Please select one or more titles.",
                                 "warning")
            self.send("XML-TEI data", None, self)
            return

        # Clear created Inputs.
        self.clearCreatedInputs()

        # Initialize progress bar.
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.selectedTitles))

        # Attempt to connect to Theatre-classique and retrieve plays...
        xml_contents = list()
        annotations = list()
        try:
            for title in self.selectedTitles:
                response = urllib.request.urlopen(
                    self.document_base_url +
                    self.filteredTitleSeg[title].annotations["url"])
                xml_contents.append(response.read().decode('utf-8'))
                annotations.append(
                    self.filteredTitleSeg[title].annotations.copy())
                progressBar.advance()  # 1 tick on the progress bar...

        # If an error occurs (e.g. http error, or memory error)...
        except:

            # Set Info box and widget to "error" state.
            self.infoBox.setText(
                "Couldn't download data from theatre-classique website.",
                "error")

            # Reset output channel.
            self.send("XML-TEI data", None, self)
            self.controlArea.setDisabled(False)
            return

        # Store downloaded XML in input objects...
        for xml_content_idx in range(len(xml_contents)):
            newInput = Input(xml_contents[xml_content_idx], self.captionTitle)
            self.createdInputs.append(newInput)

        # If there"s only one play, the widget"s output is the created Input.
        if len(self.createdInputs) == 1:
            self.segmentation = self.createdInputs[0]

        # Otherwise the widget"s output is a concatenation...
        else:
            self.segmentation = Segmenter.concatenate(
                self.createdInputs,
                self.captionTitle,
                import_labels_as=None,
            )

        # Annotate segments...
        for idx, segment in enumerate(self.segmentation):
            segment.annotations.update(annotations[idx])
            self.segmentation[idx] = segment

        # Store imported URLs as setting.
        self.importedURLs = [
            self.filteredTitleSeg[self.selectedTitles[0]].annotations["url"]
        ]

        # Set status to OK and report data size...
        message = "%i segment@p sent to output " % len(self.segmentation)
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += "(%i character@p)." % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)
        progressBar.finish()

        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)

        # Send token...
        self.send("XML-TEI data", self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
示例#27
0
    def sendData(self):
        """Compute result of widget processing and send to output"""
        # Skip if title list is empty:
        if self.myBasket == list():
            self.infoBox.setText(
                "Your corpus is empty, please add some books first", "warning")
            return

        # Clear created Inputs.
        self.clearCreatedInputs()

        self.controlArea.setDisabled(True)

        # Initialize progress bar.
        progressBar = ProgressBar(
            self,
            iterations=len(self.myBasket),
        )

        text_content = list()
        annotations = list()

        try:
            # Retrieve selected texts from gutenberg
            for text in self.myBasket:

                gutenberg_id = text[2]

                # Get the text with Gutenbergpy
                gutenberg_text = gutenbergpy.textget.strip_headers(
                    gutenbergpy.textget.get_text_by_id(gutenberg_id)).decode(
                        "utf-8")

                text_content.append(gutenberg_text)
                # populate the annotation list
                annotations.append([text[0], text[1], text[3]])
                progressBar.advance()

        # If an error occurs (e.g. http error, or memory error)...
        except Exception as exc:
            # Set Info box and widget to "error" state.
            self.infoBox.setText("Couldn't download data from Gutenberg",
                                 "error")
            self.controlArea.setDisabled(False)
            print(exc)
            return

        # Store downloaded text strings in input objects...
        for text in text_content:
            newInput = Input(text, self.captionTitle)
            self.createdInputs.append(newInput)

        # If there's only one text, the widget's output is the created Input.
        if len(self.createdInputs) == 1:
            self.segmentation = self.createdInputs[0]

        # Otherwise the widget"s output is a concatenation.
        else:
            self.segmentation = Segmenter.concatenate(
                self.createdInputs,
                self.captionTitle,
                import_labels_as=None,
            )

        # Annotate segments with book metadata
        for idx, segment in enumerate(self.segmentation):
            segment.annotations.update({"title": annotations[idx][0]})
            segment.annotations.update({"author": annotations[idx][1]})
            segment.annotations.update({"language": annotations[idx][2]})
            self.segmentation[idx] = segment

        # Clear progress bar.
        progressBar.finish()

        self.controlArea.setDisabled(False)

        # Set status to OK and report data size...
        message = "%i segment@p sent to output " % len(self.segmentation)
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += "(%i character@p)." % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)

        self.send("Gutenberg importation", self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
示例#28
0
    def sendData(self):
        """Compute result of widget processing and send to output"""

        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input", "warning")
            self.send("Linguistically analyzed data", None, self)
            return

        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait...",
            "warning",
        )
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.inputSeg))

        tokenizedSegments = list()

        # Process each input segment...
        for segment in self.inputSeg:

            # Input segment attributes...
            inputContent = segment.get_content()
            inputAnnotations = segment.annotations
            inputString = segment.str_index
            inputStart = segment.start or 0
            inputEnd = segment.end or len(inputContent)

            # NLP analysis...
            doc = self.nlp(inputContent)

            # Process each token in input segment...
            for token in doc:
                tokenAnnotations = inputAnnotations.copy()
                tokenAnnotations.update({
                    k: getattr(token, k)
                    for k in RELEVANT_KEYS if getattr(token, k) is not None
                })
                tokenStart = inputStart + token.idx
                tokenizedSegments.append(
                    Segment(
                        str_index=inputString,
                        start=tokenStart,
                        end=tokenStart + len(token),
                        annotations=tokenAnnotations,
                    ))

            progressBar.advance()

        outputSeg = Segmentation(tokenizedSegments, self.captionTitle)

        # Set status to OK and report data size...
        message = "%i segment@p sent to output." % len(outputSeg)
        message = pluralize(message, len(outputSeg))
        self.infoBox.setText(message)

        print(outputSeg.to_string())

        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)

        # Send data to output...
        self.send("Linguistically analyzed data", outputSeg, self)

        self.sendButton.resetSettingsChangedFlag()
    def sendData(self):
        """Load files, create and send segmentation"""

        # Check that there's something on input...
        if ((self.displayAdvancedSettings and not self.files)
                or not (self.file or self.displayAdvancedSettings)):
            self.infoBox.setText(u'Please select input file.', 'warning')
            self.send('Text data', None, self)
            return

        # Check that autoNumberKey is not empty (if necessary)...
        if self.displayAdvancedSettings and self.autoNumber:
            if self.autoNumberKey:
                autoNumberKey = self.autoNumberKey
            else:
                self.infoBox.setText(
                    u'Please enter an annotation key for auto-numbering.',
                    'warning')
                self.send('Text data', None, self)
                return
        else:
            autoNumberKey = None

        # Clear created Inputs...
        self.clearCreatedInputs()

        fileContents = list()
        annotations = list()
        counter = 1

        if self.displayAdvancedSettings:
            myFiles = self.files
        else:
            myFiles = [[self.file, self.encoding, "", "", "", "eng", False]]

        self.infoBox.setText(u"Processing, please wait...", "warning")
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(myFiles))

        # Open and process each file successively...
        for myFile in myFiles:
            filePath = myFile[0]
            encoding = myFile[1]
            encoding = re.sub(r"[ ]\(.+", "", encoding)
            annotation_key = myFile[2]
            annotation_value = myFile[3]
            pdf_password = myFile[4]  # SuperTextFiles
            ocr_languages = myFile[5]  # SuperTextFiles
            ocr_force = myFile[6]  # SuperTextFiles

            myFiletype = filetype.guess(myFile[0])  # SuperTextFiles

            # Try to open the file...
            self.error()
            # Start SuperTextFiles
            try:
                if myFiletype is None:
                    fileContent = self.extract_raw_text(filePath, encoding)

                elif myFiletype.extension == "pdf":
                    if ocr_force is True:
                        fileContent = self.get_pdf_content(
                            filePath,
                            ocr_languages,
                        )
                    else:
                        if self.is_textual_pdf_file(filePath) is True:
                            fileContent = self.extract_text_from_pdf(filePath)
                        else:
                            fileContent = self.get_pdf_content(
                                filePath,
                                ocr_languages,
                            )

                elif myFiletype.extension in IMG_FILETYPES:
                    fileContent = self.ocrize(filePath, ocr_languages)

                if fileContent == -1:
                    message = u"Couldn't open file."
                    self.infoBox.setText(message, 'error')
                    self.send('Text data', None, self)
                    self.controlArea.setDisabled(False)
                    return

            # End SuperTextFiles

            except IOError as e:
                if "tesseract" in str(e):
                    QMessageBox.warning(None, 'Textable', str(e),
                                        QMessageBox.Ok)
                progressBar.finish()
                if len(myFiles) > 1:
                    message = u"Couldn't open file '%s'." % filePath
                else:
                    message = u"Couldn't open file."
                self.infoBox.setText(message, 'error')
                self.send('Text data', None, self)
                self.controlArea.setDisabled(False)
                return

            # Remove utf-8 BOM if necessary...
            if encoding == u'utf-8':
                fileContent = fileContent.lstrip(
                    codecs.BOM_UTF8.decode('utf-8'))

            # Normalize text (canonical decomposition then composition)...
            fileContent = normalize('NFC', fileContent)

            fileContents.append(fileContent)

            # Annotations...
            annotation = dict()
            if self.displayAdvancedSettings:
                if annotation_key and annotation_value:
                    annotation[annotation_key] = annotation_value
                if self.importFilenames and self.importFilenamesKey:
                    filename = os.path.basename(filePath)
                    annotation[self.importFilenamesKey] = filename
                if self.autoNumber and self.autoNumberKey:
                    annotation[self.autoNumberKey] = counter
                    counter += 1
            annotations.append(annotation)
            progressBar.advance()

        # Create an LTTL.Input for each file...
        if len(fileContents) == 1:
            label = self.captionTitle
        else:
            label = None
        for index in range(len(fileContents)):
            myInput = Input(fileContents[index], label)
            segment = myInput[0]
            segment.annotations.update(annotations[index])
            myInput[0] = segment
            self.createdInputs.append(myInput)

        # If there's only one file, the widget's output is the created Input.
        if len(fileContents) == 1:
            self.segmentation = self.createdInputs[0]
        # Otherwise the widget's output is a concatenation...
        else:
            self.segmentation = Segmenter.concatenate(
                segmentations=self.createdInputs,
                label=self.captionTitle,
                copy_annotations=True,
                import_labels_as=None,
                sort=False,
                auto_number_as=None,
                merge_duplicates=False,
                progress_callback=None,
            )

        message = u'%i segment@p sent to output ' % len(self.segmentation)
        message = pluralize(message, len(self.segmentation))
        numChars = 0
        for segment in self.segmentation:
            segmentLength = len(Segmentation.get_data(segment.str_index))
            numChars += segmentLength
        message += u'(%i character@p).' % numChars
        message = pluralize(message, numChars)
        self.infoBox.setText(message)
        progressBar.finish()
        self.controlArea.setDisabled(False)

        self.send('Text data', self.segmentation, self)
        self.sendButton.resetSettingsChangedFlag()
    def sendData(self):
        """Compute result of widget processing and send to output."""

        # Check that there's a model...
        if not self.model:
            self.noLanguageModelWarning()
            self.sendNoneToOutputs()
            return

        # Check that there's an input...
        if self.inputSeg is None:
            self.infoBox.setText("Widget needs input.", "warning")
            self.sendNoneToOutputs()
            return
       
        # Initialize progress bar.
        self.infoBox.setText(
            u"Processing, please wait...", 
            "warning",
        )

        # Disable control area and initialize progress bar...
        self.controlArea.setDisabled(True)
        progressBar = ProgressBar(self, iterations=len(self.char_df))       

        # Get start and end pos of concatenated input segments...
        startPositions = [0]
        endPositions = list()
        numSegments = len(self.inputSeg)
        for idx in range(1, numSegments):
            prevSegLen = len(self.inputSeg[idx-1].get_content())
            startPositions.append(startPositions[-1] + prevSegLen + 1)
            endPositions.append(startPositions[-1] - 1)
        endPositions.append(startPositions[-1] + 
                            len(self.inputSeg[-1].get_content()) + 1)

        # Get or update character aliases...
        find_pairs = sys.modules['charnetto.find_pairs']
        characters = [entry.split(", ") for entry in self.characters]
        find_pairs.map_names(self.char_df, characters)

        # Initializations...
        charSegments = list()
        currentSegmentIdx = 0
                
        # For each character token in Charnetto's output...
        for index, charToken in self.char_df.iterrows():
        
            # Skip non-PER named entities.
            if charToken["tag"] != "PER":
                continue

            # Get index of containing segment...
            while charToken["end_pos"] > endPositions[currentSegmentIdx]:
                currentSegmentIdx += 1
                
            # Create segment for char with its actual coordinates...
            strIndex = self.inputSeg[currentSegmentIdx].str_index
            start = charToken["start_pos"]-startPositions[currentSegmentIdx]
            end = charToken["end_pos"]-startPositions[currentSegmentIdx]
            annotations = {"id": charToken["alias"]}
            charSegments.append(Segment(strIndex, start, end, annotations))
            
            progressBar.advance()

        # Send output...
        outputSegmentation = Segmentation(charSegments, 
                                           label=self.captionTitle)
        self.send("Character segmentation", outputSegmentation, self)
        print(outputSegmentation.to_string())

        # Set status to OK and report data size...
        message = "%i segment@p sent to output." % len(outputSegmentation)
        message = pluralize(message, len(outputSegmentation))
        self.infoBox.setText(message)
        
        # Clear progress bar.
        progressBar.finish()
        self.controlArea.setDisabled(False)
                
        self.sendButton.resetSettingsChangedFlag()