コード例 #1
0
    def fetch_likes_parser(self, offset=0):
        '''
        Update and save fields:
            * likes - count of likes
        Update relations:
            * like_users - users, who likes this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'likes/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 120
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 60
            post_data['offset'] = offset

        log.debug('Fetching likes of post "%s" of owner "%s", offset %d' % (self.remote_id, self.wall_owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)

        if offset == 0:
            try:
                self.likes = int(parser.content_bs.find('a', {'id': 'wk_likes_tablikes'}).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning('Strange markup of first page likes response: "%s"' % parser.content)
            self.like_users.clear()

        #<div class="wk_likes_liker_row inl_bl" id="wk_likes_liker_row722246">
        #  <div class="wk_likes_likerph_wrap" onmouseover="WkView.likesBigphOver(this, 722246)">
        #    <a class="wk_likes_liker_ph" href="/kicolenka">
        #      <img class="wk_likes_liker_img" src="http://cs418825.vk.me/v418825246/6cf8/IBbSfmDz6R8.jpg" width="100" height="100" />
        #    </a>
        #  </div>
        #  <div class="wk_likes_liker_name"><a class="wk_likes_liker_lnk" href="/kicolenka">Оля Киселева</a></div>
        #</div>

        items = parser.add_users(users=('div', {'class': re.compile(r'^wk_likes_liker_row')}),
            user_link=('a', {'class': 'wk_likes_liker_lnk'}),
            user_photo=('img', {'class': 'wk_likes_liker_img'}),
            user_add=lambda user: self.like_users.add(user))

        if len(items) == number_on_page:
            self.fetch_likes_parser(offset=offset + number_on_page)
        else:
            return self.like_users.all()
コード例 #2
0
    def fetch_reposts_parser(self, offset=0):
        '''
        OLD method via parser, may works incorrect
        Update and save fields:
            * reposts - count of reposts
        Update relations
            * repost_users - users, who repost this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'shares/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 40
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 20
            post_data['offset'] = offset

        log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' % (self.remote_id, self.wall_owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)
        if offset == 0:
            try:
                self.reposts = int(parser.content_bs.find('a', {'id': 'wk_likes_tabshares'}).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning('Strange markup of first page shares response: "%s"' % parser.content)
            self.repost_users.clear()

        #<div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
        #  <div class="post_table">
        #    <div class="post_image">
        #      <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
        #    </div>
        #      <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">

        items = parser.add_users(users=('div', {'id': re.compile('^post\d'), 'class': re.compile('^post ')}),
            user_link=('a', {'class': 'author'}),
            user_photo=lambda item: item.find('a', {'class': 'post_image'}).find('img'),
            user_add=lambda user: self.repost_users.add(user))

        if len(items) == number_on_page:
            self.fetch_reposts(offset=offset + number_on_page)
        else:
            return self.repost_users.all()
コード例 #3
0
    def fetch_group_wall_parser(self,
                                group,
                                offset=0,
                                count=None,
                                own=False,
                                after=None):
        '''
        Old method via parser
        TODO: `before` parameter not implemented
        '''
        post_data = {
            'al': 1,
            'offset': offset,
            'own': int(own),  # posts by only group or any users
            'part': 1,  # without header, footer
        }

        log.debug('Fetching post of group "%s", offset %d' % (group, offset))

        parser = VkontakteWallParser().request('/wall-%s' % group.remote_id,
                                               data=post_data)

        items = parser.content_bs.findAll(
            'div', {
                'class': re.compile('^post'),
                'id': re.compile('^post-%d' % group.remote_id)
            })

        current_count = offset + len(items)
        need_cut = count and count < current_count
        if need_cut:
            items = items[:count - offset]

        for item in items:

            try:
                post = parser.parse_post(item, group)
            except VkontakteParseError, e:
                log.error(e)
                continue

            if after and post.date < after:
                need_cut = True
                break

            post.raw_html = unicode(item)
            post.save()
            parsed.send(sender=Post, instance=post, container=item)
コード例 #4
0
    def fetch_group_post_parser(
            self,
            post,
            offset=0,
            count=None):  # jkj, after=None, only_new=False):
        '''
        Old method via parser
        '''
        post_data = {
            'al': 1,
            'offset': offset,
            'part': 1,
        }

        log.debug('Fetching comments to post "%s" of owner "%s", offset %d' %
                  (post.remote_id, post.wall_owner, offset))

        parser = VkontakteWallParser().request('/wall%s' % (post.remote_id),
                                               data=post_data)

        items = parser.content_bs.findAll('div', {'class': 'fw_reply'})

        current_count = offset + len(items)
        need_cut = count and count < current_count
        if need_cut:
            items = items[:count - offset]

#        # get date of last comment and set after attribute
#        if only_new:
#            comments = post.wall_comments.order_by('-date')
#            if comments:
#                after = comments[0].date

        for item in items:

            try:
                comment = parser.parse_comment(item, post.wall_owner)
            except VkontakteParseError, e:
                log.error(e)
                continue

            comment.post = post
            comment.raw_html = unicode(item)
            comment.save()
            parsed.send(sender=Comment, instance=comment, container=item)
コード例 #5
0
    def fetch_group_post_parser(self, post, offset=0, count=None):  # jkj, after=None, only_new=False):
        '''
        Old method via parser
        '''
        post_data = {
            'al': 1,
            'offset': offset,
            'part': 1,
        }

        log.debug('Fetching comments to post "%s" of owner "%s", offset %d' % (post.remote_id, post.wall_owner, offset))

        parser = VkontakteWallParser().request('/wall%s' % (post.remote_id), data=post_data)

        items = parser.content_bs.findAll('div', {'class': 'fw_reply'})

        current_count = offset + len(items)
        need_cut = count and count < current_count
        if need_cut:
            items = items[:count - offset]

#        # get date of last comment and set after attribute
#        if only_new:
#            comments = post.wall_comments.order_by('-date')
#            if comments:
#                after = comments[0].date

        for item in items:

            try:
                comment = parser.parse_comment(item, post.wall_owner)
            except VkontakteParseError, e:
                log.error(e)
                continue

            comment.post = post
            comment.raw_html = unicode(item)
            comment.save()
            parsed.send(sender=Comment, instance=comment, container=item)
コード例 #6
0
    def fetch_group_wall_parser(self, group, offset=0, count=None, own=False, after=None):
        '''
        Old method via parser
        TODO: `before` parameter not implemented
        '''
        post_data = {
            'al': 1,
            'offset': offset,
            'own': int(own),  # posts by only group or any users
            'part': 1,  # without header, footer
        }

        log.debug('Fetching post of group "%s", offset %d' % (group, offset))

        parser = VkontakteWallParser().request('/wall-%s' % group.remote_id, data=post_data)

        items = parser.content_bs.findAll('div', {'class': re.compile('^post'), 'id': re.compile('^post-%d' % group.remote_id)})

        current_count = offset + len(items)
        need_cut = count and count < current_count
        if need_cut:
            items = items[:count - offset]

        for item in items:

            try:
                post = parser.parse_post(item, group)
            except VkontakteParseError, e:
                log.error(e)
                continue

            if after and post.date < after:
                need_cut = True
                break

            post.raw_html = unicode(item)
            post.save()
            parsed.send(sender=Post, instance=post, container=item)
コード例 #7
0
    def fetch_reposts_parser(self, offset=0):
        '''
        OLD method via parser, may works incorrect
        Update and save fields:
            * reposts - count of reposts
        Update relations
            * repost_users - users, who repost this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'shares/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 40
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 20
            post_data['offset'] = offset

        log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' %
                  (self.remote_id, self.wall_owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)
        if offset == 0:
            try:
                self.reposts = int(
                    parser.content_bs.find('a', {
                        'id': 'wk_likes_tabshares'
                    }).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning(
                    'Strange markup of first page shares response: "%s"' %
                    parser.content)
            self.repost_users.clear()

        #<div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
        #  <div class="post_table">
        #    <div class="post_image">
        #      <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
        #    </div>
        #      <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">

        items = parser.add_users(
            users=('div', {
                'id': re.compile('^post\d'),
                'class': re.compile('^post ')
            }),
            user_link=('a', {
                'class': 'author'
            }),
            user_photo=lambda item: item.find('a', {
                'class': 'post_image'
            }).find('img'),
            user_add=lambda user: self.repost_users.add(user))

        if len(items) == number_on_page:
            self.fetch_reposts(offset=offset + number_on_page)
        else:
            return self.repost_users.all()
コード例 #8
0
    def fetch_likes_parser(self, offset=0):
        '''
        Update and save fields:
            * likes - count of likes
        Update relations:
            * like_users - users, who likes this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'likes/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 120
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 60
            post_data['offset'] = offset

        log.debug('Fetching likes of post "%s" of owner "%s", offset %d' %
                  (self.remote_id, self.wall_owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)

        if offset == 0:
            try:
                self.likes = int(
                    parser.content_bs.find('a', {
                        'id': 'wk_likes_tablikes'
                    }).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning(
                    'Strange markup of first page likes response: "%s"' %
                    parser.content)
            self.like_users.clear()

        #<div class="wk_likes_liker_row inl_bl" id="wk_likes_liker_row722246">
        #  <div class="wk_likes_likerph_wrap" onmouseover="WkView.likesBigphOver(this, 722246)">
        #    <a class="wk_likes_liker_ph" href="/kicolenka">
        #      <img class="wk_likes_liker_img" src="http://cs418825.vk.me/v418825246/6cf8/IBbSfmDz6R8.jpg" width="100" height="100" />
        #    </a>
        #  </div>
        #  <div class="wk_likes_liker_name"><a class="wk_likes_liker_lnk" href="/kicolenka">Оля Киселева</a></div>
        #</div>

        items = parser.add_users(
            users=('div', {
                'class': re.compile(r'^wk_likes_liker_row')
            }),
            user_link=('a', {
                'class': 'wk_likes_liker_lnk'
            }),
            user_photo=('img', {
                'class': 'wk_likes_liker_img'
            }),
            user_add=lambda user: self.like_users.add(user))

        if len(items) == number_on_page:
            self.fetch_likes_parser(offset=offset + number_on_page)
        else:
            return self.like_users.all()