def fetch_likes_parser(self, offset=0):
'''
Update and save fields:
* likes - count of likes
Update relations:
* like_users - users, who likes this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'likes/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 120
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 60
post_data['offset'] = offset
log.debug('Fetching likes of post "%s" of owner "%s", offset %d' % (self.remote_id, self.wall_owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.likes = int(parser.content_bs.find('a', {'id': 'wk_likes_tablikes'}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning('Strange markup of first page likes response: "%s"' % parser.content)
self.like_users.clear()
#<div class="wk_likes_liker_row inl_bl" id="wk_likes_liker_row722246">
# <div class="wk_likes_likerph_wrap" onmouseover="WkView.likesBigphOver(this, 722246)">
# <a class="wk_likes_liker_ph" href="/kicolenka">
# <img class="wk_likes_liker_img" src="http://cs418825.vk.me/v418825246/6cf8/IBbSfmDz6R8.jpg" width="100" height="100" />
# </a>
# </div>
# <div class="wk_likes_liker_name"><a class="wk_likes_liker_lnk" href="/kicolenka">Оля Киселева</a></div>
#</div>
items = parser.add_users(users=('div', {'class': re.compile(r'^wk_likes_liker_row')}),
user_link=('a', {'class': 'wk_likes_liker_lnk'}),
user_photo=('img', {'class': 'wk_likes_liker_img'}),
user_add=lambda user: self.like_users.add(user))
if len(items) == number_on_page:
self.fetch_likes_parser(offset=offset + number_on_page)
else:
return self.like_users.all()
def fetch_reposts_parser(self, offset=0):
'''
OLD method via parser, may works incorrect
Update and save fields:
* reposts - count of reposts
Update relations
* repost_users - users, who repost this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'shares/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 40
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 20
post_data['offset'] = offset
log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' % (self.remote_id, self.wall_owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.reposts = int(parser.content_bs.find('a', {'id': 'wk_likes_tabshares'}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning('Strange markup of first page shares response: "%s"' % parser.content)
self.repost_users.clear()
#<div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
# <div class="post_table">
# <div class="post_image">
# <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
# </div>
# <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">
items = parser.add_users(users=('div', {'id': re.compile('^post\d'), 'class': re.compile('^post ')}),
user_link=('a', {'class': 'author'}),
user_photo=lambda item: item.find('a', {'class': 'post_image'}).find('img'),
user_add=lambda user: self.repost_users.add(user))
if len(items) == number_on_page:
self.fetch_reposts(offset=offset + number_on_page)
else:
return self.repost_users.all()
def fetch_group_wall_parser(self,
group,
offset=0,
count=None,
own=False,
after=None):
'''
Old method via parser
TODO: `before` parameter not implemented
'''
post_data = {
'al': 1,
'offset': offset,
'own': int(own), # posts by only group or any users
'part': 1, # without header, footer
}
log.debug('Fetching post of group "%s", offset %d' % (group, offset))
parser = VkontakteWallParser().request('/wall-%s' % group.remote_id,
data=post_data)
items = parser.content_bs.findAll(
'div', {
'class': re.compile('^post'),
'id': re.compile('^post-%d' % group.remote_id)
})
current_count = offset + len(items)
need_cut = count and count < current_count
if need_cut:
items = items[:count - offset]
for item in items:
try:
post = parser.parse_post(item, group)
except VkontakteParseError, e:
log.error(e)
continue
if after and post.date < after:
need_cut = True
break
post.raw_html = unicode(item)
post.save()
parsed.send(sender=Post, instance=post, container=item)
def fetch_group_post_parser(
self,
post,
offset=0,
count=None): # jkj, after=None, only_new=False):
'''
Old method via parser
'''
post_data = {
'al': 1,
'offset': offset,
'part': 1,
}
log.debug('Fetching comments to post "%s" of owner "%s", offset %d' %
(post.remote_id, post.wall_owner, offset))
parser = VkontakteWallParser().request('/wall%s' % (post.remote_id),
data=post_data)
items = parser.content_bs.findAll('div', {'class': 'fw_reply'})
current_count = offset + len(items)
need_cut = count and count < current_count
if need_cut:
items = items[:count - offset]
# # get date of last comment and set after attribute
# if only_new:
# comments = post.wall_comments.order_by('-date')
# if comments:
# after = comments[0].date
for item in items:
try:
comment = parser.parse_comment(item, post.wall_owner)
except VkontakteParseError, e:
log.error(e)
continue
comment.post = post
comment.raw_html = unicode(item)
comment.save()
parsed.send(sender=Comment, instance=comment, container=item)
def fetch_group_post_parser(self, post, offset=0, count=None): # jkj, after=None, only_new=False):
'''
Old method via parser
'''
post_data = {
'al': 1,
'offset': offset,
'part': 1,
}
log.debug('Fetching comments to post "%s" of owner "%s", offset %d' % (post.remote_id, post.wall_owner, offset))
parser = VkontakteWallParser().request('/wall%s' % (post.remote_id), data=post_data)
items = parser.content_bs.findAll('div', {'class': 'fw_reply'})
current_count = offset + len(items)
need_cut = count and count < current_count
if need_cut:
items = items[:count - offset]
# # get date of last comment and set after attribute
# if only_new:
# comments = post.wall_comments.order_by('-date')
# if comments:
# after = comments[0].date
for item in items:
try:
comment = parser.parse_comment(item, post.wall_owner)
except VkontakteParseError, e:
log.error(e)
continue
comment.post = post
comment.raw_html = unicode(item)
comment.save()
parsed.send(sender=Comment, instance=comment, container=item)
def fetch_group_wall_parser(self, group, offset=0, count=None, own=False, after=None):
'''
Old method via parser
TODO: `before` parameter not implemented
'''
post_data = {
'al': 1,
'offset': offset,
'own': int(own), # posts by only group or any users
'part': 1, # without header, footer
}
log.debug('Fetching post of group "%s", offset %d' % (group, offset))
parser = VkontakteWallParser().request('/wall-%s' % group.remote_id, data=post_data)
items = parser.content_bs.findAll('div', {'class': re.compile('^post'), 'id': re.compile('^post-%d' % group.remote_id)})
current_count = offset + len(items)
need_cut = count and count < current_count
if need_cut:
items = items[:count - offset]
for item in items:
try:
post = parser.parse_post(item, group)
except VkontakteParseError, e:
log.error(e)
continue
if after and post.date < after:
need_cut = True
break
post.raw_html = unicode(item)
post.save()
parsed.send(sender=Post, instance=post, container=item)
def fetch_reposts_parser(self, offset=0):
'''
OLD method via parser, may works incorrect
Update and save fields:
* reposts - count of reposts
Update relations
* repost_users - users, who repost this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'shares/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 40
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 20
post_data['offset'] = offset
log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' %
(self.remote_id, self.wall_owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.reposts = int(
parser.content_bs.find('a', {
'id': 'wk_likes_tabshares'
}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning(
'Strange markup of first page shares response: "%s"' %
parser.content)
self.repost_users.clear()
#<div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
# <div class="post_table">
# <div class="post_image">
# <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
# </div>
# <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">
items = parser.add_users(
users=('div', {
'id': re.compile('^post\d'),
'class': re.compile('^post ')
}),
user_link=('a', {
'class': 'author'
}),
user_photo=lambda item: item.find('a', {
'class': 'post_image'
}).find('img'),
user_add=lambda user: self.repost_users.add(user))
if len(items) == number_on_page:
self.fetch_reposts(offset=offset + number_on_page)
else:
return self.repost_users.all()
def fetch_likes_parser(self, offset=0):
'''
Update and save fields:
* likes - count of likes
Update relations:
* like_users - users, who likes this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'likes/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 120
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 60
post_data['offset'] = offset
log.debug('Fetching likes of post "%s" of owner "%s", offset %d' %
(self.remote_id, self.wall_owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.likes = int(
parser.content_bs.find('a', {
'id': 'wk_likes_tablikes'
}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning(
'Strange markup of first page likes response: "%s"' %
parser.content)
self.like_users.clear()
#<div class="wk_likes_liker_row inl_bl" id="wk_likes_liker_row722246">
# <div class="wk_likes_likerph_wrap" onmouseover="WkView.likesBigphOver(this, 722246)">
# <a class="wk_likes_liker_ph" href="/kicolenka">
# <img class="wk_likes_liker_img" src="http://cs418825.vk.me/v418825246/6cf8/IBbSfmDz6R8.jpg" width="100" height="100" />
# </a>
# </div>
# <div class="wk_likes_liker_name"><a class="wk_likes_liker_lnk" href="/kicolenka">Оля Киселева</a></div>
#</div>
items = parser.add_users(
users=('div', {
'class': re.compile(r'^wk_likes_liker_row')
}),
user_link=('a', {
'class': 'wk_likes_liker_lnk'
}),
user_photo=('img', {
'class': 'wk_likes_liker_img'
}),
user_add=lambda user: self.like_users.add(user))
if len(items) == number_on_page:
self.fetch_likes_parser(offset=offset + number_on_page)
else:
return self.like_users.all()