def getUserReviews(self, user):
        logging.info( 'Get user reviews for %d id: %s:', user.gid, user.id)

        cookies=dict(session_id2='',
        p='',
        u='')
        url = 'https://www.goodreads.com/review/list/'+str(user.gid)+'-stephanie?order=d&per_page=100&shelf=read&sort=rating&utf8=✓&view=reviews'
        r = requests.get(url, cookies=cookies)
        if r.status_code != 200:
            logging.warning('Invalid request: %s' % r.status_code)
            return (None, False)
        else:
            soup = BeautifulSoup(r.content, 'html.parser')
            if(soup.find('div', attrs={'id': 'privateProfile'})):
                logging.warning('User %d is private', user.gid)
                return (None, False)
            divPagination=soup.find('div', attrs={'id':'reviewPagination'})
            if(divPagination is None):
                reviews, books, authors=parseUserReviews(self, r.content, users_to_user(user))
                return ({'reviews': reviews, 'books': books, 'authors': authors}, False)
            pages_a=divPagination.findAll('a')
            pages=[]
            for a in pages_a:
                class_a=a.get('class')
                if(class_a is None or(class_a[0]!='next_page' and class_a[0]!= 'previous_page')):
                    pages.append({'num':int(a.text), 'link': a.get('href')})
            all_reviews=[]
            all_books=[]
            all_authors=[]

            reviews, books, authors=parseUserReviews(self, r.content, users_to_user(user))
            all_reviews.append(reviews)
            all_books.append(books)
            all_authors.append(authors)
            if(len(pages)>1):
                total_pages_count=pages[-1]['num']+1
                if(total_pages_count>6):
                    total_pages_count=6
                for p in range(2,total_pages_count):
                    reviews, books, authors= self.getPageReviews(users_to_user(user), p, total_pages_count)
                    all_reviews.append(reviews)
                    all_books.append(books)
                    all_authors.append(authors)
            all_reviews = [val for sublist in all_reviews for val in sublist]
            all_books=[val for sublist in all_books for val in sublist]
            all_authors=[val for sublist in all_authors for val in sublist]
            logging.info( '\nFinish getting %d reviews %d books %d authors',len(all_reviews), len(all_books), len(all_authors))
            return ({'reviews': all_reviews, 'books': all_books, 'authors': all_authors}, False)
 def getPageReviews(self, user, page,total):
     logging.info('get review for user '+str(user.gid)+' page '+str(page)+' of '+str(total))
     cookies=dict(session_id2='',
     p='',
     u='-6IY_')
     url = 'https://www.goodreads.com/review/list/'+str(user.gid)+'-stephanie?order=d&page='+str(page)+'&per_page=100&sort=rating&shelf=read&utf8=✓&view=reviews'
     try:
         r = requests.get(url, cookies=cookies)
         if r.status_code != 200:
             logging.warning('Invalid request: %s' % r.status_code)
             return ([],[], [])
         else:
             return parseUserReviews(self, r.content, user)
     except Exception as e:
         logging.error('Exception on page reviews')
         return ([],[], [])
Exemple #3
0
 def getPageReviews(self, user, page, total):
     logging.info('get review for user ' + str(user.gid) + ' page ' +
                  str(page) + ' of ' + str(total))
     cookies = dict(session_id2='', p='', u='-6IY_')
     url = 'https://www.goodreads.com/review/list/' + str(
         user.gid) + '-stephanie?order=d&page=' + str(
             page
         ) + '&per_page=100&sort=rating&shelf=read&utf8=✓&view=reviews'
     try:
         r = requests.get(url, cookies=cookies)
         if r.status_code != 200:
             logging.warning('Invalid request: %s' % r.status_code)
             return ([], [], [])
         else:
             return parseUserReviews(self, r.content, user)
     except Exception as e:
         logging.error('Exception on page reviews')
         return ([], [], [])
Exemple #4
0
    def getUserReviews(self, user):
        logging.info('Get user reviews for %d id: %s:', user.gid, user.id)

        cookies = dict(session_id2='', p='', u='')
        url = 'https://www.goodreads.com/review/list/' + str(
            user.gid
        ) + '-stephanie?order=d&per_page=100&shelf=read&sort=rating&utf8=✓&view=reviews'
        r = requests.get(url, cookies=cookies)
        if r.status_code != 200:
            logging.warning('Invalid request: %s' % r.status_code)
            return (None, False)
        else:
            soup = BeautifulSoup(r.content, 'html.parser')
            if (soup.find('div', attrs={'id': 'privateProfile'})):
                logging.warning('User %d is private', user.gid)
                return (None, False)
            divPagination = soup.find('div', attrs={'id': 'reviewPagination'})
            if (divPagination is None):
                reviews, books, authors = parseUserReviews(
                    self, r.content, users_to_user(user))
                return ({
                    'reviews': reviews,
                    'books': books,
                    'authors': authors
                }, False)
            pages_a = divPagination.findAll('a')
            pages = []
            for a in pages_a:
                class_a = a.get('class')
                if (class_a is None or (class_a[0] != 'next_page'
                                        and class_a[0] != 'previous_page')):
                    pages.append({'num': int(a.text), 'link': a.get('href')})
            all_reviews = []
            all_books = []
            all_authors = []

            reviews, books, authors = parseUserReviews(self, r.content,
                                                       users_to_user(user))
            all_reviews.append(reviews)
            all_books.append(books)
            all_authors.append(authors)
            if (len(pages) > 1):
                total_pages_count = pages[-1]['num'] + 1
                if (total_pages_count > 6):
                    total_pages_count = 6
                for p in range(2, total_pages_count):
                    reviews, books, authors = self.getPageReviews(
                        users_to_user(user), p, total_pages_count)
                    all_reviews.append(reviews)
                    all_books.append(books)
                    all_authors.append(authors)
            all_reviews = [val for sublist in all_reviews for val in sublist]
            all_books = [val for sublist in all_books for val in sublist]
            all_authors = [val for sublist in all_authors for val in sublist]
            logging.info('\nFinish getting %d reviews %d books %d authors',
                         len(all_reviews), len(all_books), len(all_authors))
            return ({
                'reviews': all_reviews,
                'books': all_books,
                'authors': all_authors
            }, False)