Exemplo n.º 1
0
    def run(self, data):

        """Performs a scraping of a faculty members GoogleScholar page.
        :param data is a faculty object
        :return: last faculty member handled
        """
        
        no_text_count = 0
        for faculty in data:
            faculty_name = faculty.name
    
            search_results = Faculty.search().query('match', name=faculty_name).execute()
            if len(search_results) > 1:
                # Shouldn't happen, but could.
                raise WorkflowException("Professor id is ambiguous during search ... More than 1 result")

            search_dup = Document.search().query('match', faculty_id=faculty.faculty_id).query("match", source="GoogleScholar")
            search_dup.delete()

            faculty = search_results[0]
            if faculty.google_scholar is not None and "http" in faculty.google_scholar:
                scraper = ScraperFactory.create_scraper(faculty.google_scholar, ScraperType.GOOGLESCHOLAR)
                scrapps = scraper.get_scrapps()
                for scrapp in scrapps:
                    doc = Document()
                    doc.source = "GoogleScholar"
                    doc.faculty_id = faculty.faculty_id
                    doc.text = scrapp.title
                    doc.save()
            else:
                no_text_count += 1
        print("NO TEXT COUNT = ", no_text_count)
        return faculty
Exemplo n.º 2
0
    def run_next(self):
        """Runs the next task in the workflow.

        :exception WorkflowTaskArgumentException: If a task receives unsatisfactory data.
        :returns False if there are no more steps to be run, True if the step succeeds.
        """
        if self.current_step >= self.steps:
            return False
        """Fetch the class of the next task, and instantiate it"""
        current_task = self.get_current_task()()

        if current_task.is_requirement_satisfied(self.last_result):
            try:
                result = current_task.run(self.last_result)
            except ScraperException:
                return False

            self.last_result = result
            self.current_step += 1
        else:
            raise WorkflowException(
                "{} received an unsatisfactory argument - {}".format(
                    current_task.task_name, self.last_result))

        return True
Exemplo n.º 3
0
    def run(self, data):
        """Performs a scraping of a faculty members ResearchId page.
        :param data is a faculty object
        :return: last faculty member handled
        """

        no_text_count = 0
        for faculty in data:
            faculty_name = faculty.name

            search_results = Faculty.search().query(
                'match', name=faculty_name).execute()
            if len(search_results) > 1:
                # Shouldn't happen, but could.
                raise WorkflowException(
                    "Professor id is ambiguous during search ... More than 1 result"
                )

            search_dup = Document.search().query(
                'match',
                faculty_id=faculty.faculty_id).query("match",
                                                     source="ResearchId")
            search_dup.delete()
            faculty = search_results[0]
            if faculty.research_id is not None:

                scraper = ScraperFactory.create_scraper(
                    faculty.research_id, ScraperType.RESEARCHID)
                scrapps = scraper.get_scrapps()

                keywords_and_description = scrapps[0]
                titles = scrapps[1:]

                doc = Document()
                doc.faculty_id = faculty.faculty_id
                doc.source = "ResearchId"
                try:
                    doc.text = keywords_and_description.meta_data[
                        "description"]
                except:
                    print("No description")
                    doc.text = ""
                try:
                    doc.user_keywords = keywords_and_description.meta_data[
                        "keywords"]
                except:
                    print("No keywords")
                doc.save()

                for scrapp in titles:
                    doc = Document()
                    doc.source = "ResearchId"
                    doc.faculty_id = faculty.faculty_id
                    doc.text = scrapp.title
                    doc.save()

            else:
                no_text_count += 1
        print("NO TEXT COUNT = ", no_text_count)
        return faculty
Exemplo n.º 4
0
    def run(self, data):
        """Performs a scraping of a faculty members GoogleScholar page.
        :param data is a faculty object
        :return: list of faculty members
        """

        faculty = data
        if isinstance(faculty, str):
            search_results = Faculty.search().query(
                'match', name=faculty_name).execute()
            if len(search_results) > 1:
                # Shouldn't happen, but could.
                raise WorkflowException(
                    "Professor id is ambiguous during search ... More than 1 result"
                )
            faculty = search_results[0]
        faculty_name = faculty.name

        Document.search().query('match', faculty_id=faculty.faculty_id) \
            .query("match", source="GoogleScholar") \
            .delete()

        if faculty.google_scholar is not None and "http" in faculty.google_scholar:
            scraper = ScraperFactory.create_scraper(faculty.google_scholar,
                                                    ScraperType.GOOGLESCHOLAR)
            scrapps = scraper.get_scrapps()
            for scrapp in scrapps:
                doc = Document()
                doc.source = "GoogleScholar"
                doc.faculty_id = faculty.faculty_id
                doc.text = scrapp.title
                doc.save()

        return faculty
Exemplo n.º 5
0
    def run(self, data):
        """Performs a scraping of a faculty members ResearchId page.
        :param data is a faculty object
        :return: last faculty member handled
        """

        faculty = data
        if isinstance(faculty, str):
            search_results = Faculty.search().query('match',
                                                    name=faculty).execute()
            if len(search_results) > 1:
                # Shouldn't happen, but could.
                raise WorkflowException(
                    "Professor id is ambiguous during search ... More than 1 result"
                )
            faculty = search_results[0]

        faculty_name = faculty.name


        Document.search().query('match', faculty_id=faculty.faculty_id) \
            .query("match", source="ResearchId") \
            .delete()

        print("Running researchid scrape on {}. Research id {}.".format(
            faculty_name, faculty.research_id))

        if faculty.research_id is not None:

            scraper = ScraperFactory.create_scraper(faculty.research_id,
                                                    ScraperType.RESEARCHID)
            scrapps = scraper.get_scrapps()

            keywords_and_description = scrapps[0]
            titles = scrapps[1:]

            doc = Document()
            doc.faculty_id = faculty.faculty_id
            doc.source = "ResearchId"
            try:
                doc.text = keywords_and_description.meta_data["description"]
            except:
                print("No description")
                doc.text = ""
            try:
                doc.user_keywords = keywords_and_description.meta_data[
                    "keywords"]
            except:
                print("No keywords")
            doc.save()

            for scrapp in titles:
                doc = Document()
                doc.source = "ResearchId"
                doc.faculty_id = faculty.faculty_id
                doc.text = scrapp.title
                doc.save()

        return faculty
Exemplo n.º 6
0
    def run(self, data):
        """Updates a Faculty members information in Elasticsearch, based on the result of a scrape.

        :param data: list of tuples of form <str, Scrapp>
        :return: The updated instance of a Faculty model.
        """

        faculty_name = data[0]
        scrapp = data[1]

        search_results = Faculty.search().query('match',
                                                name=faculty_name).execute()
        if len(search_results) > 1:
            # Shouldn't happen, but could.
            raise WorkflowException(
                "Faculty name is ambiguous during search... More than 1 result"
            )

        faculty = search_results[0]

        Document.search().query('match', faculty_id=faculty.faculty_id) \
            .query("match", source="Profile") \
            .delete()

        if "orcid_link" in scrapp.meta_data:
            faculty.orc_id = scrapp.meta_data["orcid_link"]

        if "researchid_link" in scrapp.meta_data:
            faculty.research_id = scrapp.meta_data["researchid_link"]

        if "googlescholar_link" in scrapp.meta_data:
            faculty.google_scholar = scrapp.meta_data["googlescholar_link"]

        if "text" in scrapp.meta_data:
            doc_search = Document.search().query('match', faculty_id=faculty.faculty_id) \
                .query('match', source = "profile") \
                .execute()
            try:
                doc = doc_search[0]
            except IndexError:
                doc = Document()
                doc.faculty_id = faculty.faculty_id
                doc.source = "profile"

            doc.text = scrapp.meta_data["text"]
            doc.date = datetime.now()
            doc.save()

        faculty.save()

        return faculty