class UserableModelMixin(models.Model):
class Meta:
abstract = True
if 'vkontakte_users' in settings.INSTALLED_APPS:
from vkontakte_users.models import User
members = ManyToManyHistoryField(User,
related_name='members_%(class)ss',
versions=True)
@atomic
def update_members(self, *args, **kwargs):
ids = self.__class__.remote.get_members_ids(group=self,
*args,
**kwargs)
count = len(ids)
initial = self.members.versions.count() == 0
self.members = ids
# update members_count
if self.members_count != count:
self.members_count = count
self.save()
if initial:
self.members.get_query_set_through().update(time_from=None)
self.members.versions.update(added_count=0)
return self.members
else:
members = get_improperly_configured_field('vkontakte_users', True)
update_members = get_improperly_configured_field('vkontakte_users')
class Status(TwitterBaseModel):
author = models.ForeignKey('User', related_name='statuses')
text = models.TextField()
favorited = models.BooleanField(default=False)
retweeted = models.BooleanField(default=False)
truncated = models.BooleanField(default=False)
source = models.CharField(max_length=100)
source_url = models.URLField(null=True)
favorites_count = models.PositiveIntegerField()
retweets_count = models.PositiveIntegerField()
replies_count = models.PositiveIntegerField(null=True)
in_reply_to_status = models.ForeignKey('Status', null=True, related_name='replies')
in_reply_to_user = models.ForeignKey('User', null=True, related_name='replies')
favorites_users = ManyToManyHistoryField('User', related_name='favorites')
retweeted_status = models.ForeignKey('Status', null=True, related_name='retweets')
place = fields.JSONField(null=True)
# format the next fields doesn't clear
contributors = fields.JSONField(null=True)
coordinates = fields.JSONField(null=True)
geo = fields.JSONField(null=True)
objects = models.Manager()
remote = StatusManager(methods={
'get': 'get_status',
})
def __unicode__(self):
return u'%s: %s' % (self.author, self.text)
@property
def slug(self):
return '/%s/status/%d' % (self.author.screen_name, self.id)
def parse(self):
self._response['favorites_count'] = self._response.pop('favorite_count', 0)
self._response['retweets_count'] = self._response.pop('retweet_count', 0)
self._response.pop('user', None)
self._response.pop('in_reply_to_screen_name', None)
self._response.pop('in_reply_to_user_id_str', None)
self._response.pop('in_reply_to_status_id_str', None)
self._get_foreignkeys_for_fields('in_reply_to_status', 'in_reply_to_user')
super(Status, self).parse()
def fetch_retweets(self, **kwargs):
return Status.remote.fetch_retweets(status=self, **kwargs)
def fetch_replies(self, **kwargs):
return Status.remote.fetch_replies(status=self, **kwargs)
class LikableModelMixin(models.Model):
likes_users = ManyToManyHistoryField(User, related_name='like_%(class)ss')
likes_count = models.PositiveIntegerField(u'Likes',
null=True,
db_index=True)
class Meta:
abstract = True
@property
def likes_remote_type(self):
raise NotImplementedError()
@atomic
def fetch_likes(self, *args, **kwargs):
kwargs['likes_type'] = self.likes_remote_type
kwargs['item_id'] = self.remote_id_short
kwargs['owner_id'] = self.owner_remote_id
log.debug('Fetching likes of %s %s of owner "%s"' %
(self._meta.module_name, self.remote_id, self.owner))
ids = User.remote.fetch_likes_user_ids(*args, **kwargs)
self.likes_users = User.remote.fetch(ids=ids, only_expired=True)
# update self.likes_count
likes_count = self.likes_users.count()
if likes_count < self.likes_count:
log.warning(
'Fetched ammount of like users less, than attribute `likes` of post "%s": %d < %d'
% (self.remote_id, likes_count, self.likes_count))
elif likes_count > self.likes_count:
self.likes_count = likes_count
self.save()
return self.likes_users.all()
def parse(self, response):
if 'likes' in response:
value = response.pop('likes')
if isinstance(value, int):
response['likes_count'] = value
elif isinstance(value, dict) and 'count' in value:
response['likes_count'] = value['count']
super(LikableModelMixin, self).parse(response)
class LikableModelMixin(models.Model):
likes_users = ManyToManyHistoryField(User, related_name='like_%(class)ss')
likes_count = models.PositiveIntegerField(null=True, help_text='The number of likes of this item')
class Meta:
abstract = True
def parse(self, response):
if 'like_count' in response:
response['likes_count'] = response.pop('like_count')
super(LikableModelMixin, self).parse(response)
def update_count_and_get_like_users(self, instances, *args, **kwargs):
self.likes_users = instances
self.likes_count = instances.count()
self.save()
return instances
# TODO: commented, becouse if many processes fetch_likes, got errors
# DatabaseError: deadlock detected
# DETAIL: Process 27235 waits for ShareLock on transaction 3922627359; blocked by process 27037.
# @atomic
@fetch_all(return_all=update_count_and_get_like_users, paging_next_arg_name='after')
def fetch_likes(self, limit=1000, **kwargs):
"""
Retrieve and save all likes of post
"""
ids = []
response = api_call('%s/likes' % self.graph_id, limit=limit, **kwargs)
if response:
log.debug('response objects count=%s, limit=%s, after=%s' %
(len(response['data']), limit, kwargs.get('after')))
for resource in response['data']:
try:
user = get_or_create_from_small_resource(resource)
ids += [user.pk]
except UnknownResourceType:
continue
return User.objects.filter(pk__in=ids), response
class Album(PhotoBase):
class Meta:
verbose_name = u'Альбом фотографий Одноклассники'
verbose_name_plural = u'Альбомы фотографий Одноклассники'
remote_pk_field = 'aid'
created = models.DateField(null=True)
like_users = ManyToManyHistoryField(User, related_name='like_albums')
owner_content_type = models.ForeignKey(
ContentType, related_name='odnoklassniki_albums_owners')
owner_id = models.BigIntegerField(db_index=True)
owner = generic.GenericForeignKey('owner_content_type', 'owner_id')
photos_count = models.PositiveIntegerField(default=0)
title = models.TextField()
remote = AlbumRemoteManager(
methods={
'get': 'getAlbums',
'get_one': 'getAlbumInfo',
'get_likes': 'getAlbumLikes',
})
@property
def slug(self):
return '%s/album/%s' % (self.owner.slug, self.id)
def __unicode__(self):
return self.id
def fetch_photos(self, **kwargs):
return Photo.remote.fetch(group=self.owner, album=self, **kwargs)
def fetch_likes(self, **kwargs):
kwargs['aid'] = self.pk
return super(Album, self).fetch_likes(**kwargs)
class ReactionableModelMixin(models.Model):
# without "Like": it may broke something
reaction_types = ['love', 'wow', 'haha', 'sad', 'angry', 'thankful']
reactions_count = models.PositiveIntegerField(null=True, help_text='The number of reactions of this item')
def update_count_and_get_users_builder(reaction):
def update_count_and_get_reaction_users(self, instances, *args, **kwargs):
# setattr(self, '{0}s_count'.format(reaction), 0)
setattr(self, '{0}s_users'.format(reaction), instances)
setattr(self, '{0}s_count'.format(reaction), instances.count())
self.save()
return instances
return update_count_and_get_reaction_users
for reaction in reaction_types:
related_name = '%s_' % reaction + '%(class)ss'
vars()['{0}s_users'.format(reaction)] = ManyToManyHistoryField(User, related_name=related_name)
vars()['{0}s_count'.format(reaction)] = models.PositiveIntegerField(null=True, help_text='The number of {0}s of this item'.format(reaction))
vars()['update_count_and_get_{0}_users'.format(reaction)] = update_count_and_get_users_builder(reaction=reaction)
class Meta:
abstract = True
def parse(self, response):
for reaction in self.reaction_types:
if '{0}_count'.format(reaction) in response:
response['{0}s_count'.format(reaction)] = response.pop('{0}_count'.format(reaction))
super(ReactionableModelMixin, self).parse(response)
def fetch_reactions(self, reaction=None, limit=1000, **kwargs):
"""
Retrieve and save all reactions of post
Note: method may return different data structures:
List: if reaction is specified
Dictionary: if reaction is not specified
"""
ids = {}
types = self.reaction_types + ['LIKE']
for id_type in types:
ids[id_type.upper()] = []
response = api_call('%s/reactions' % self.graph_id, version=2.6, limit=limit, **kwargs)
if response:
log.debug('response objects count=%s, limit=%s, after=%s' %
(len(response['data']), limit, kwargs.get('after')))
for resource in response['data']:
try:
if (reaction != None) and (reaction.upper() != resource['type']):
continue
try:
user = get_or_create_from_small_resource(resource)
ids[resource['type']] += [user.pk]
except UnknownResourceType:
continue
# no 'type' in resource
except KeyError:
continue
def get_user_ids(self, ids, response):
return User.objects.filter(pk__in=ids), response
result = {}
for id_type in types:
if (reaction != None) and (reaction.upper() != id_type.upper()):
continue
count_method = getattr(self, 'update_count_and_get_{0}_users'.format(id_type.lower()))
# create count-and-get function wrapped in fetch_all decorator
fetch = fetch_all(return_all=count_method, paging_next_arg_name='after')(get_user_ids)
result[id_type.upper()] = fetch(self, ids[id_type.upper()], response)
# for some reason fetch_all does not call count method
count_method(result[id_type.upper()])
if (reaction != None):
return result[reaction.upper()]
else:
return result
# separate from fetch method, because it would return wrong data if reaction specified
def count_reactions(self):
count = 0
for reaction in self.reaction_types + ['like']:
count += getattr(self, '{0}s_count'.format(reaction))
self.reactions_count = count
self.save()
class User(TwitterBaseModel):
screen_name = models.CharField(u'Screen name', max_length=50, unique=True)
name = models.CharField(u'Name', max_length=100)
description = models.TextField(u'Description')
location = models.CharField(u'Location', max_length=100)
time_zone = models.CharField(u'Time zone', max_length=100, null=True)
contributors_enabled = models.BooleanField(u'Contributors enabled', default=False)
default_profile = models.BooleanField(u'Default profile', default=False)
default_profile_image = models.BooleanField(u'Default profile image', default=False)
follow_request_sent = models.BooleanField(u'Follow request sent', default=False)
following = models.BooleanField(u'Following', default=False)
geo_enabled = models.BooleanField(u'Geo enabled', default=False)
is_translator = models.BooleanField(u'Is translator', default=False)
notifications = models.BooleanField(u'Notifications', default=False)
profile_use_background_image = models.BooleanField(u'Profile use background image', default=False)
protected = models.BooleanField(u'Protected', default=False)
verified = models.BooleanField(u'Verified', default=False)
profile_background_image_url = models.URLField(max_length=300, null=True)
profile_background_image_url_https = models.URLField(max_length=300, null=True)
profile_background_tile = models.BooleanField(default=False)
profile_background_color = models.CharField(max_length=6)
profile_banner_url = models.URLField(max_length=300, null=True)
profile_image_url = models.URLField(max_length=300, null=True)
profile_image_url_https = models.URLField(max_length=300)
url = models.URLField(max_length=300, null=True)
profile_link_color = models.CharField(max_length=6)
profile_sidebar_border_color = models.CharField(max_length=6)
profile_sidebar_fill_color = models.CharField(max_length=6)
profile_text_color = models.CharField(max_length=6)
favorites_count = models.PositiveIntegerField()
followers_count = models.PositiveIntegerField()
friends_count = models.PositiveIntegerField()
listed_count = models.PositiveIntegerField()
statuses_count = models.PositiveIntegerField()
utc_offset = models.IntegerField(null=True)
followers = ManyToManyHistoryField('User', versions=True)
objects = models.Manager()
remote = UserManager(methods={
'get': 'get_user',
})
def __unicode__(self):
return self.name
def save(self, *args, **kwargs):
if self.friends_count < 0:
log.warning('Negative value friends_count=%s set to 0 for user ID %s' % (self.friends_count, self.id))
self.friends_count = 0
super(User, self).save(*args, **kwargs)
@property
def slug(self):
return self.screen_name
def parse(self):
self._response['favorites_count'] = self._response.pop('favourites_count', None)
self._response.pop('status', None)
super(User, self).parse()
def fetch_followers(self, **kwargs):
return User.remote.fetch_followers_for_user(user=self, **kwargs)
def get_followers_ids(self, **kwargs):
return User.remote.get_followers_ids_for_user(user=self, **kwargs)
def fetch_statuses(self, **kwargs):
return Status.remote.fetch_for_user(user=self, **kwargs)
class ShareableModelMixin(models.Model):
shares_users = ManyToManyHistoryField(User, related_name='shares_%(class)ss')
shares_count = models.PositiveIntegerField(null=True, help_text='The number of shares of this item')
class Meta:
abstract = True
def update_count_and_get_shares_users(self, instances, *args, **kwargs):
# self.shares_users = instances
# becouse here are not all shares: "Some posts may not appear here because of their privacy settings."
if self.shares_count is None:
self.shares_count = instances.count()
self.save()
return instances
@atomic
@fetch_all(return_all=update_count_and_get_shares_users, paging_next_arg_name='after')
def fetch_shares(self, limit=1000, **kwargs):
"""
Retrieve and save all shares of post
"""
from facebook_api.models import MASTER_DATABASE # here, becouse cycling import
ids = []
response = api_call('%s/sharedposts' % self.graph_id, **kwargs)
if response:
timestamps = dict(
[(int(post['from']['id']), datetime_parse(post['created_time'])) for post in response['data']])
ids_new = timestamps.keys()
# becouse we should use local pk, instead of remote, remove it after pk -> graph_id
ids_current = map(int, User.objects.filter(pk__in=self.shares_users.get_query_set(
only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)).values_list('graph_id', flat=True))
ids_add = set(ids_new).difference(set(ids_current))
ids_add_pairs = []
ids_remove = set(ids_current).difference(set(ids_new))
log.debug('response objects count=%s, limit=%s, after=%s' %
(len(response['data']), limit, kwargs.get('after')))
for post in response['data']:
graph_id = int(post['from']['id'])
if sorted(post['from'].keys()) == ['id', 'name']:
try:
user = get_or_create_from_small_resource(post['from'])
ids += [user.pk]
# this id in add list and still not in add_pairs (sometimes in response are duplicates)
if graph_id in ids_add and graph_id not in map(lambda i: i[0], ids_add_pairs):
# becouse we should use local pk, instead of remote
ids_add_pairs += [(graph_id, user.pk)]
except UnknownResourceType:
continue
m2m_model = self.shares_users.through
# '(album|post)_id'
field_name = [f.attname for f in m2m_model._meta.local_fields
if isinstance(f, models.ForeignKey) and f.name != 'user'][0]
# remove old shares without time_from
self.shares_users.get_query_set_through().filter(time_from=None).delete()
# in case some ids_add already left
self.shares_users.get_query_set_through().filter(
**{field_name: self.pk, 'user_id__in': map(lambda i: i[1], ids_add_pairs)}).delete()
# add new shares with specified `time_from` value
get_share_date = lambda id: timestamps[id] if id in timestamps else self.created_time
m2m_model.objects.bulk_create([m2m_model(
**{field_name: self.pk, 'user_id': pk, 'time_from': get_share_date(graph_id)}) for graph_id, pk in ids_add_pairs])
return User.objects.filter(pk__in=ids), response
class Media(InstagramBaseModel):
remote_id = models.CharField(max_length=30, unique=True)
caption = models.TextField(blank=True)
link = models.URLField(max_length=68)
type = models.CharField(max_length=5)
filter = models.CharField(max_length=40) # TODO: tune max_length of this field
image_low_resolution = models.URLField(max_length=200)
image_standard_resolution = models.URLField(max_length=200)
image_thumbnail = models.URLField(max_length=200)
video_low_bandwidth = models.URLField(max_length=130)
video_low_resolution = models.URLField(max_length=130)
video_standard_resolution = models.URLField(max_length=130)
created_time = models.DateTimeField()
comments_count = models.PositiveIntegerField(null=True)
likes_count = models.PositiveIntegerField(null=True)
user = models.ForeignKey('User', related_name="media_feed")
location = models.ForeignKey('Location', null=True, related_name="media_feed")
likes_users = ManyToManyHistoryField('User', related_name="likes_media")
tags = models.ManyToManyField('Tag', related_name='media_feed')
remote = MediaManager(remote_pk=('remote_id',), methods={
'get': 'media',
'user_recent_media': 'user_recent_media',
'tag_recent_media': 'tag_recent_media',
'location_recent_media': 'location_recent_media',
})
def get_url(self):
return self.link
def __unicode__(self):
return self.caption
def parse(self):
self._response['remote_id'] = self._response.pop('id')
for prefix in ['video', 'image']:
key = '%ss' % prefix
if key in self._response:
for k, v in self._response[key].items():
media = self._response[key][k]
if isinstance(media, ApiModel):
media = media.__dict__
self._response['%s_%s' % (prefix, k)] = media['url']
if not isinstance(self._response['created_time'], datetime):
self._response['created_time'] = timestamp_to_datetime(self._response['created_time'])
if 'comment_count' in self._response:
self._response['comments_count'] = self._response.pop('comment_count')
elif 'comments' in self._response:
self._response['comments_count'] = self._response.pop('comments')['count']
if 'like_count' in self._response:
self._response['likes_count'] = self._response.pop('like_count')
elif 'likes' in self._response:
self._response['likes_count'] = self._response.pop('likes')['count']
if isinstance(self._response['caption'], ApiModel):
self._response['caption'] = self._response['caption'].text
elif isinstance(self._response['caption'], dict):
self._response['caption'] = self._response['caption']['text']
# if 'likes' in self._response:
# self._response['likes_users'] = self._response.pop('likes')
super(Media, self).parse()
def fetch_comments(self):
return Comment.remote.fetch_media_comments(self)
def fetch_likes(self):
return User.remote.fetch_media_likes(self)
def save(self, *args, **kwargs):
if self.caption is None:
self.caption = ''
super(Media, self).save(*args, **kwargs)
for field, relations in self._relations_post_save['fk'].items():
extra_fields = {'media_id': self.pk, 'owner_id': self.user_id} if field == 'comments' else {}
for instance in relations:
instance.__dict__.update(extra_fields)
instance.__class__.remote.get_or_create_from_instance(instance)
for field, relations in self._relations_post_save['m2m'].items():
for instance in relations:
instance = instance.__class__.remote.get_or_create_from_instance(instance)
getattr(self, field).add(instance)
class User(InstagramBaseModel):
id = models.BigIntegerField(primary_key=True)
username = models.CharField(max_length=30, unique=True)
full_name = models.CharField(max_length=30)
bio = models.CharField(max_length=150)
profile_picture = models.URLField(max_length=112)
website = models.URLField(max_length=150) # found max_length=106
followers_count = models.PositiveIntegerField(null=True, db_index=True)
follows_count = models.PositiveIntegerField(null=True, db_index=True)
media_count = models.PositiveIntegerField(null=True, db_index=True)
followers = ManyToManyHistoryField('User', versions=True, related_name='follows')
is_private = models.NullBooleanField('Account is private', db_index=True)
objects = models.Manager()
remote = UserManager(methods={
'get': 'user',
'search': 'user_search',
'follows': 'user_follows',
'followers': 'user_followed_by',
'likes': 'media_likes',
})
@property
def slug(self):
return self.username
def __unicode__(self):
return self.full_name or self.username
@property
def instagram_link(self):
return u'https://instagram.com/%s/' % self.username
def _substitute(self, old_instance):
super(User, self)._substitute(old_instance)
for field_name in ['followers_count', 'follows_count', 'media_count', 'is_private']:
if getattr(self, field_name) is None and getattr(old_instance, field_name) is not None:
setattr(self, field_name, getattr(old_instance, field_name))
def save(self, *args, **kwargs):
# cut all CharFields to max allowed length
for field in self._meta.local_fields:
if isinstance(field, models.CharField):
setattr(self, field.name, getattr(self, field.name)[:field.max_length])
try:
with atomic():
super(InstagramModel, self).save(*args, **kwargs)
except IntegrityError as e:
if 'username' in e.message:
# duplicate key value violates unique constraint "instagram_api_user_username_key"
# DETAIL: Key (username)=(...) already exists.
user_local = User.objects.get(username=self.username)
try:
# check for recursive loop
# get remote user
user_remote = User.remote.get(user_local.pk)
try:
user_local2 = User.objects.get(username=user_remote.username)
# if users excahnge usernames or user is dead (400 error)
if user_local2.pk == self.pk or user_remote.is_private:
user_local.username = '******' % time.time()
user_local.save()
except User.DoesNotExist:
pass
# fetch right user
User.remote.fetch(user_local.pk)
except InstagramError as e:
if e.code == 400:
user_local.delete()
else:
raise
super(InstagramModel, self).save(*args, **kwargs)
else:
raise
def parse(self):
if isinstance(self._response, dict) and 'counts' in self._response:
count = self._response['counts']
self._response['followers_count'] = count.get('followed_by', 0)
self._response['follows_count'] = count.get('follows', 0)
self._response['media_count'] = count.get('media', 0)
super(User, self).parse()
def fetch_follows(self):
return User.remote.fetch_follows(user=self)
def fetch_followers(self):
return User.remote.fetch_followers(user=self)
def fetch_media(self, **kwargs):
return Media.remote.fetch_user_media(user=self, **kwargs)
def refresh(self):
# do refresh via client_id, because is_private is dependent on access_token and relation with current user
with override_api_context('instagram', use_client_id=True):
super(User, self).refresh()
class Media(InstagramBaseModel):
remote_id = models.CharField(max_length=100, unique=True)
caption = models.TextField(blank=True)
link = models.URLField(max_length=300)
type = models.CharField(max_length=20)
image_low_resolution = models.URLField(max_length=200)
image_standard_resolution = models.URLField(max_length=200)
image_thumbnail = models.URLField(max_length=200)
video_low_bandwidth = models.URLField(max_length=200)
video_low_resolution = models.URLField(max_length=200)
video_standard_resolution = models.URLField(max_length=200)
created_time = models.DateTimeField()
comments_count = models.PositiveIntegerField(null=True)
likes_count = models.PositiveIntegerField(null=True)
user = models.ForeignKey(User, related_name="media_feed")
likes_users = ManyToManyHistoryField('User', related_name="likes_media")
remote = MediaManager(remote_pk=('remote_id',), methods={
'get': 'media',
'user_recent_media': 'user_recent_media',
'tag_recent_media': 'tag_recent_media',
})
def get_url(self):
return self.link
def __unicode__(self):
return self.caption
def parse(self):
self._response['remote_id'] = self._response.pop('id')
for prefix in ['video', 'image']:
key = '%ss' % prefix
if key in self._response:
for k, v in self._response[key].items():
media = self._response[key][k]
if isinstance(media, ApiModel):
media = media.__dict__
self._response['%s_%s' % (prefix, k)] = media['url']
if not isinstance(self._response['created_time'], datetime):
self._response['created_time'] = timestamp_to_datetime(self._response['created_time'])
if 'comment_count' in self._response:
self._response['comments_count'] = self._response.pop('comment_count')
elif 'comments' in self._response:
self._response['comments_count'] = self._response.pop('comments')['count']
if 'like_count' in self._response:
self._response['likes_count'] = self._response.pop('like_count')
elif 'likes' in self._response:
self._response['likes_count'] = self._response.pop('likes')['count']
if isinstance(self._response['caption'], ApiModel):
self._response['caption'] = self._response['caption'].text
elif isinstance(self._response['caption'], dict):
self._response['caption'] = self._response['caption']['text']
super(Media, self).parse()
def fetch_comments(self):
return Comment.remote.fetch_media_comments(self)
def fetch_likes(self):
return User.remote.fetch_media_likes(self)
def save(self, *args, **kwargs):
if self.caption is None:
self.caption = ''
super(Media, self).save(*args, **kwargs)
class RepostableModelMixin(models.Model):
reposts_users = ManyToManyHistoryField(User,
related_name='reposts_%(class)ss')
reposts_count = models.PositiveIntegerField(u'Кол-во репостов',
null=True,
db_index=True)
class Meta:
abstract = True
def parse(self, response):
if 'reposts' in response:
value = response.pop('reposts')
if isinstance(value, int):
response['reposts_count'] = value
elif isinstance(value, dict) and 'count' in value:
response['reposts_count'] = value['count']
super(RepostableModelMixin, self).parse(response)
@property
def reposters(self):
return [repost.author for repost in self.wall_reposts.all()]
def fetch_reposts(self, source='api', *args, **kwargs):
if source == 'api':
return self.fetch_reposts_api(*args, **kwargs)
else:
return self.fetch_reposts_parser(*args, **kwargs)
def fetch_reposts_api(self, *args, **kwargs):
self.fetch_instance_reposts(*args, **kwargs)
# update self.reposts_count
reposts_count = self.reposts_users.get_query_set(only_pk=True).count()
if reposts_count < self.reposts_count:
log.warning(
'Fetched ammount of repost users less, than attribute `reposts` of post "%s": %d < %d'
% (self.remote_id, reposts_count, self.reposts_count))
elif reposts_count > self.reposts_count:
self.reposts_count = reposts_count
self.save()
return self.reposts_users.all()
@atomic
def fetch_instance_reposts(self, *args, **kwargs):
resources = self.fetch_reposts_items(*args, **kwargs)
if not resources:
return self.__class__.objects.none()
# TODO: still complicated to store reposts objects, may be it's task for another application
# posts = Post.remote.parse_response(resources, extra_fields={'copy_post_id': self.pk})
# Post.objects.filter(pk__in=set([Post.remote.get_or_create_from_instance(instance).pk
# for instance in posts]))
# positive ids -> only users
# TODO: think about how to store reposts by groups
timestamps = dict([(post['from_id'], post['date'])
for post in resources if post['from_id'] > 0])
ids_new = timestamps.keys()
ids_current = self.reposts_users.get_query_set(
only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)
ids_current_left = self.reposts_users.get_query_set_through().using(MASTER_DATABASE).exclude(time_to=None) \
.values_list('user_id', flat=True)
ids_add = set(ids_new).difference(set(ids_current))
ids_remove = set(ids_current).difference(set(ids_new))
# some of them may be already left for some reason or API error
ids_unleft = set(ids_add).intersection(set(ids_current_left))
ids_add = ids_add.difference(ids_unleft)
# fetch new users
User.remote.fetch(ids=ids_add, only_expired=True)
# remove old reposts without time_from
self.reposts_users.get_query_set_through().filter(
time_from=None).delete()
# try to find left users, that present in ids_add and make them unleft
self.reposts_users.get_query_set_through().exclude(
time_to=None).filter(user_id__in=ids_unleft).update(time_to=None)
# add new reposts
get_repost_date = lambda id: datetime.utcfromtimestamp(timestamps[
id]).replace(tzinfo=timezone.utc
) if id in timestamps else self.date
m2m_model = self.reposts_users.through
m2m_model.objects.bulk_create([
m2m_model(
**{
'user_id': id,
'post_id': self.pk,
'time_from': get_repost_date(id)
}) for id in ids_add
])
# remove reposts
self.reposts_users.get_query_set_through().filter(
user_id__in=ids_remove).update(time_to=timezone.now())
# не рекомендуется указывать default_count из-за бага паджинации репостов: https://vk.com/wall-51742963_6860
@fetch_all(max_extra_calls=3)
def fetch_reposts_items(self, offset=0, count=1000, *args, **kwargs):
if count > 1000:
raise ValueError("Parameter 'count' can not be more than 1000")
# owner_id
# идентификатор пользователя или сообщества, на стене которого находится запись. Если параметр не задан, то он считается равным идентификатору текущего пользователя.
# Обратите внимание, идентификатор сообщества в параметре owner_id необходимо указывать со знаком "-" — например, owner_id=-1 соответствует идентификатору сообщества ВКонтакте API (club1)
kwargs['owner_id'] = self.owner_remote_id
# post_id
# идентификатор записи на стене.
kwargs['post_id'] = self.remote_id_short
# offset
# смещение, необходимое для выборки определенного подмножества записей.
kwargs['offset'] = int(offset)
# count
# количество записей, которое необходимо получить.
# положительное число, по умолчанию 20, максимальное значение 100
kwargs['count'] = int(count)
response = api_call('wall.getReposts', **kwargs)
log.debug(
'Fetching reposts for post %s: %d returned, offset %d, count %d' %
(self.remote_id, len(response['items']), offset, count))
return response['items']
@atomic
def fetch_reposts_parser(self, offset=0):
'''
OLD method via parser, may works incorrect
Update and save fields:
* reposts - count of reposts
Update relations
* reposts_users - users, who repost this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'shares/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 40
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 20
post_data['offset'] = offset
log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' %
(self.remote_id, self.owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.reposts_count = int(
parser.content_bs.find('a', {
'id': 'wk_likes_tabshares'
}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning(
'Strange markup of first page shares response: "%s"' %
parser.content)
self.reposts_users.clear()
# <div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
# <div class="post_table">
# <div class="post_image">
# <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
# </div>
# <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">
items = parser.add_users(
users=('div', {
'id': re.compile('^post\d'),
'class': re.compile('^post ')
}),
user_link=('a', {
'class': 'author'
}),
user_photo=lambda item: item.find('a', {
'class': 'post_image'
}).find('img'),
user_add=lambda user: self.reposts_users.add(user))
if len(items) == number_on_page:
self.fetch_reposts(offset=offset + number_on_page)
else:
return self.reposts_users.all()
class Comment(OdnoklassnikiModel):
methods_namespace = 'discussions'
# temporary variable for distance from parse() to save()
author_type = None
id = models.CharField(max_length=68, primary_key=True)
discussion = models.ForeignKey(Discussion, related_name='comments')
# denormalization for query optimization
owner_content_type = models.ForeignKey(
ContentType, related_name='odnoklassniki_comments_owners')
owner_id = models.BigIntegerField(db_index=True)
owner = generic.GenericForeignKey('owner_content_type', 'owner_id')
author_content_type = models.ForeignKey(
ContentType, related_name='odnoklassniki_comments_authors')
author_id = models.BigIntegerField(db_index=True)
author = generic.GenericForeignKey('author_content_type', 'author_id')
reply_to_comment = models.ForeignKey(
'self', null=True, verbose_name=u'Это ответ на комментарий')
reply_to_author_content_type = models.ForeignKey(
ContentType,
null=True,
related_name='odnoklassniki_comments_reply_to_authors')
reply_to_author_id = models.BigIntegerField(db_index=True, null=True)
reply_to_author = generic.GenericForeignKey('reply_to_author_content_type',
'reply_to_author_id')
object_type = models.CharField(max_length=20, choices=COMMENT_TYPE_CHOICES)
text = models.TextField()
date = models.DateTimeField()
likes_count = models.PositiveIntegerField(default=0)
liked_it = models.BooleanField(default=False)
attrs = JSONField(null=True)
like_users = ManyToManyHistoryField(User, related_name='like_comments')
remote = CommentRemoteManager(
methods={
'get': 'getComments',
'get_one': 'getComment',
'get_likes': 'getCommentLikes',
})
class Meta:
verbose_name = _('Odnoklassniki comment')
verbose_name_plural = _('Odnoklassniki comments')
@property
def slug(self):
return self.discussion.slug
def save(self, *args, **kwargs):
self.owner = self.discussion.owner
if self.author_id and not self.author:
if self.author_type == 'GROUP':
if self.author_id == self.owner_id:
self.author = self.owner
else:
from odnoklassniki_groups.models import Group
try:
self.author = Group.remote.fetch(
ids=[self.author_id])[0]
except IndexError:
raise Exception(
"Can't fetch Odnoklassniki comment's group-author with ID %s"
% self.author_id)
else:
try:
self.author = User.objects.get(pk=self.author_id)
except User.DoesNotExist:
try:
self.author = User.remote.fetch(
ids=[self.author_id])[0]
except IndexError:
raise Exception(
"Can't fetch Odnoklassniki comment's user-author with ID %s"
% self.author_id)
# it's hard to get proper reply_to_author_content_type in case we fetch comments from last
if self.reply_to_author_id and not self.reply_to_author_content_type:
self.reply_to_author_content_type = ContentType.objects.get_for_model(
User)
# if self.reply_to_comment_id and self.reply_to_author_id and not self.reply_to_author_content_type:
# try:
# self.reply_to_author = User.objects.get(pk=self.reply_to_author_id)
# except User.DoesNotExist:
# self.reply_to_author = self.reply_to_comment.author
# check for existing comment from self.reply_to_comment to prevent ItegrityError
if self.reply_to_comment_id:
try:
self.reply_to_comment = Comment.objects.get(
pk=self.reply_to_comment_id)
except Comment.DoesNotExist:
log.error(
"Try to save comment ID=%s with reply_to_comment_id=%s that doesn't exist in DB"
% (self.id, self.reply_to_comment_id))
self.reply_to_comment = None
return super(Comment, self).save(*args, **kwargs)
def parse(self, response):
# rename becouse discussion has object_type
if 'type' in response:
response['object_type'] = response.pop('type')
if 'like_count' in response:
response['likes_count'] = response.pop('like_count')
if 'reply_to_id' in response:
response['reply_to_author_id'] = response.pop('reply_to_id')
if 'reply_to_comment_id' in response:
response['reply_to_comment'] = response.pop('reply_to_comment_id')
# if author is a group
if 'author_type' in response:
response.pop('author_name')
self.author_type = response.pop('author_type')
return super(Comment, self).parse(response)
def update_likes_count(self, instances, *args, **kwargs):
users = User.objects.filter(pk__in=instances)
self.like_users = users
self.likes_count = len(instances)
self.save()
return users
@atomic
@fetch_all(return_all=update_likes_count, has_more=None)
def fetch_likes(self, count=100, **kwargs):
kwargs['comment_id'] = self.id
kwargs['discussionId'] = self.discussion.id
kwargs['discussionType'] = self.discussion.object_type
kwargs['count'] = int(count)
# kwargs['fields'] = Comment.remote.get_request_fields('user')
response = Comment.remote.api_call(method='get_likes', **kwargs)
# has_more not in dict and we need to handle pagination manualy
if 'users' not in response:
response.pop('anchor', None)
users_ids = []
else:
users_ids = list(
User.remote.get_or_create_from_resources_list(
response['users']).values_list('pk', flat=True))
return users_ids, response
class Discussion(OdnoklassnikiPKModel):
methods_namespace = ''
remote_pk_field = 'object_id'
owner_content_type = models.ForeignKey(
ContentType, related_name='odnoklassniki_discussions_owners')
owner_id = models.BigIntegerField(db_index=True)
owner = generic.GenericForeignKey('owner_content_type', 'owner_id')
author_content_type = models.ForeignKey(
ContentType, related_name='odnoklassniki_discussions_authors')
author_id = models.BigIntegerField(db_index=True)
author = generic.GenericForeignKey('author_content_type', 'author_id')
object_type = models.CharField(max_length=20,
choices=DISCUSSION_TYPE_CHOICES,
default=DISCUSSION_TYPE_DEFAULT)
title = models.TextField()
message = models.TextField()
date = models.DateTimeField(db_index=True)
last_activity_date = models.DateTimeField(null=True)
last_user_access_date = models.DateTimeField(null=True)
new_comments_count = models.PositiveIntegerField(default=0)
comments_count = models.PositiveIntegerField(default=0)
likes_count = models.PositiveIntegerField(default=0)
reshares_count = models.PositiveIntegerField(default=0)
# vote
last_vote_date = models.DateTimeField(null=True)
votes_count = models.PositiveIntegerField(default=0)
question = models.TextField()
liked_it = models.BooleanField(default=False)
entities = JSONField(null=True)
ref_objects = JSONField(null=True)
attrs = JSONField(null=True)
like_users = ManyToManyHistoryField(User, related_name='like_discussions')
remote = DiscussionRemoteManager(
methods={
'get': 'discussions.getList',
'get_one': 'discussions.get',
'get_likes': 'discussions.getDiscussionLikes',
'stream': 'stream.get',
'mget': 'mediatopic.getByIds',
})
# def __unicode__(self):
# return self.name
class Meta:
verbose_name = _('Odnoklassniki discussion')
verbose_name_plural = _('Odnoklassniki discussions')
def _substitute(self, old_instance):
super(Discussion, self)._substitute(old_instance)
try:
if self.entities['themes'][0]['images'][0] is None:
self.entities['themes'][0]['images'][
0] = old_instance.entities['themes'][0]['images'][0]
except (KeyError, TypeError):
pass
def save(self, *args, **kwargs):
# make 2 dicts {id: instance} for group and users from entities
if self.entities:
entities = {
'users': [],
'groups': [],
}
for resource in self.entities.get('users', []):
entities['users'] += [
User.remote.get_or_create_from_resource(resource)
]
for resource in self.entities.get('groups', []):
from odnoklassniki_groups.models import Group
entities['groups'] += [
Group.remote.get_or_create_from_resource(resource)
]
for field in ['users', 'groups']:
entities[field] = dict([(instance.id, instance)
for instance in entities[field]])
# set owner
if self.ref_objects:
for resource in self.ref_objects:
id = int(resource['id'])
if resource['type'] == 'GROUP':
self.owner = entities['groups'][id]
elif resource['type'] == 'USER':
self.owner = entities['user'][id]
else:
log.warning(
"Strange type of object in ref_objects %s for duscussion ID=%s"
% (resource, self.id))
# set author
if self.author_id:
if self.author_id in entities['groups']:
self.author = entities['groups'][self.author_id]
elif self.author_id in entities['users']:
self.author = entities['users'][self.author_id]
else:
log.warning(
"Imposible to find author with ID=%s in entities of duscussion ID=%s"
% (self.author_id, self.id))
self.author_id = None
if self.owner and not self.author_id:
# of no author_id (owner_uid), so it's equal to owner from ref_objects
self.author = self.owner
if self.author_id and not self.author:
self.author = self.author_content_type.model_class(
).objects.get_or_create(pk=self.author_id)[0]
if self.owner_id and not self.owner:
self.owner = self.owner_content_type.model_class(
).objects.get_or_create(pk=self.owner_id)[0]
return super(Discussion, self).save(*args, **kwargs)
@property
def refresh_kwargs(self):
return {
'id': self.id,
'type': self.object_type or DISCUSSION_TYPE_DEFAULT
}
@property
def slug(self):
return '%s/topic/%s' % (self.owner.slug, self.id)
def parse(self, response):
from odnoklassniki_groups.models import Group
if 'discussion' in response:
response.update(response.pop('discussion'))
# Discussion.remote.fetch_one
if 'entities' in response and 'media_topics' in response['entities'] \
and len(response['entities']['media_topics']) == 1:
response.update(response['entities'].pop('media_topics')[0])
if 'polls' in response['entities']:
response.update(response['entities'].pop('polls')[0])
if 'vote_summary' in response:
response['last_vote_date'] = response['vote_summary'][
'last_vote_date_ms'] / 1000
response['votes_count'] = response['vote_summary']['count']
# media_topics
if 'like_summary' in response:
response['likes_count'] = response['like_summary']['count']
response.pop('like_summary')
if 'reshare_summary' in response:
response['reshares_count'] = response['reshare_summary']['count']
response.pop('reshare_summary')
if 'discussion_summary' in response:
response['comments_count'] = response['discussion_summary'][
'comments_count']
response.pop('discussion_summary')
if 'author_ref' in response:
i = response.pop('author_ref').split(':')
response['author_id'] = i[1]
self.author_content_type = ContentType.objects.get(
app_label='odnoklassniki_%ss' % i[0], model=i[0])
if 'owner_ref' in response:
i = response.pop('owner_ref').split(':')
response['owner_id'] = i[1]
self.owner_content_type = ContentType.objects.get(
app_label='odnoklassniki_%ss' % i[0], model=i[0])
if 'created_ms' in response:
response['date'] = response.pop('created_ms') / 1000
if 'media' in response:
response['title'] = response['media'][0]['text']
# in API owner is author
if 'owner_uid' in response:
response['author_id'] = response.pop('owner_uid')
# some name cleaning
if 'like_count' in response:
response['likes_count'] = response.pop('like_count')
if 'total_comments_count' in response:
response['comments_count'] = response.pop('total_comments_count')
if 'creation_date' in response:
response['date'] = response.pop('creation_date')
# response of stream.get has another format
if 'message' in response and '{media_topic' in response['message']:
regexp = r'{media_topic:?(\d+)?}'
m = re.findall(regexp, response['message'])
if len(m):
response['id'] = m[0]
response['message'] = re.sub(regexp, '', response['message'])
return super(Discussion, self).parse(response)
def fetch_comments(self, **kwargs):
return Comment.remote.fetch(discussion=self, **kwargs)
def update_likes_count(self, instances, *args, **kwargs):
users = User.objects.filter(pk__in=instances)
self.like_users = users
self.likes_count = len(instances)
self.save()
return users
@atomic
@fetch_all(return_all=update_likes_count, has_more=None)
def fetch_likes(self, count=100, **kwargs):
kwargs['discussionId'] = self.id
kwargs['discussionType'] = self.object_type
kwargs['count'] = int(count)
# kwargs['fields'] = Discussion.remote.get_request_fields('user')
response = Discussion.remote.api_call(method='get_likes', **kwargs)
# has_more not in dict and we need to handle pagination manualy
if 'users' not in response:
response.pop('anchor', None)
users_ids = []
else:
users_ids = list(
User.remote.get_or_create_from_resources_list(
response['users']).values_list('pk', flat=True))
return users_ids, response
class Comment(WallAbstractModel):
class Meta:
verbose_name = u'Коментарий сообщения Вконтакте'
verbose_name_plural = u'Комментарии сообщений Вконтакте'
remote_pk_field = 'cid'
likes_type = 'comment'
fields_required_for_update = ['comment_id', 'post_id', 'owner_id']
post = models.ForeignKey(Post,
verbose_name=u'Пост',
related_name='wall_comments')
# Владелец стены сообщения User or Group (декомпозиция от self.post для фильтра в админке и быстрых запросов)
wall_owner_content_type = models.ForeignKey(
ContentType, related_name='vkontakte_wall_comments')
wall_owner_id = models.PositiveIntegerField(db_index=True)
wall_owner = generic.GenericForeignKey('wall_owner_content_type',
'wall_owner_id')
# Автор комментария
author_content_type = models.ForeignKey(ContentType,
related_name='comments')
author_id = models.PositiveIntegerField(db_index=True)
author = generic.GenericForeignKey('author_content_type', 'author_id')
from_id = models.IntegerField(
null=True) # strange value, seems to be equal to author
# Это ответ пользователю
reply_for_content_type = models.ForeignKey(ContentType,
null=True,
related_name='replies')
reply_for_id = models.PositiveIntegerField(null=True, db_index=True)
reply_for = generic.GenericForeignKey('reply_for_content_type',
'reply_for_id')
reply_to = models.ForeignKey('self',
null=True,
verbose_name=u'Это ответ на комментарий')
# abstract field for correct deleting group and user models in admin
group = generic.GenericForeignKey('author_content_type', 'author_id')
user = generic.GenericForeignKey('author_content_type', 'author_id')
group_wall_reply = generic.GenericForeignKey('reply_for_content_type',
'reply_for_id')
user_wall_reply = generic.GenericForeignKey('reply_for_content_type',
'reply_for_id')
date = models.DateTimeField(u'Время комментария', db_index=True)
text = models.TextField(u'Текст комментария')
likes = models.PositiveIntegerField(u'Кол-во лайков',
default=0,
db_index=True)
like_users = ManyToManyHistoryField(User, related_name='like_comments')
objects = VkontakteCRUDManager()
remote = CommentRemoteManager(remote_pk=('remote_id', ),
methods={
'get': 'getComments',
'create': 'addComment',
'update': 'editComment',
'delete': 'deleteComment',
'restore': 'restoreComment',
})
def save(self, *args, **kwargs):
self.wall_owner = self.post.wall_owner
return super(Comment, self).save(*args, **kwargs)
def prepare_create_params(self, **kwargs):
kwargs.update({
'owner_id':
self.remote_owner_id,
'post_id':
self.post.remote_id_short,
'text':
self.text,
'reply_to_comment':
self.reply_for.id if self.reply_for else '',
'from_group':
int(kwargs.get('from_group', 0)),
'attachments':
kwargs.get('attachments', ''),
})
return kwargs
def prepare_update_params(self, **kwargs):
kwargs.update({
'owner_id': self.remote_owner_id,
'comment_id': self.remote_id_short,
'message': self.text,
'attachments': kwargs.get('attachments', ''),
})
return kwargs
def prepare_delete_params(self):
return {
'owner_id': self.remote_owner_id,
'comment_id': self.remote_id_short
}
def parse_remote_id_from_response(self, response):
if response:
return '%s_%s' % (self.remote_owner_id, response['cid'])
return None
def parse(self, response):
self.raw_json = response
super(Comment, self).parse(response)
if '_' not in str(self.remote_id):
self.remote_id = '%s_%s' % (self.post.remote_id.split('_')[0],
self.remote_id)
for field_name in ['likes']:
if field_name in response and 'count' in response[field_name]:
setattr(self, field_name, response.pop(field_name)['count'])
self.author = User.objects.get_or_create(remote_id=response['uid'])[0]
if 'reply_to_uid' in response:
self.reply_for = User.objects.get_or_create(
remote_id=response['reply_to_uid'])[0]
if 'reply_to_cid' in response:
try:
self.reply_to = Comment.objects.get(
remote_id=response['reply_to_cid'])
except:
pass
class Post(WallAbstractModel):
class Meta:
verbose_name = u'Сообщение Вконтакте'
verbose_name_plural = u'Сообщения Вконтакте'
likes_type = 'post'
fields_required_for_update = ['post_id', 'owner_id']
# Владелец стены сообщения User or Group
wall_owner_content_type = models.ForeignKey(
ContentType, related_name='vkontakte_wall_posts')
wall_owner_id = models.PositiveIntegerField(db_index=True)
wall_owner = generic.GenericForeignKey('wall_owner_content_type',
'wall_owner_id')
# Создатель/автор сообщения
author_content_type = models.ForeignKey(ContentType,
related_name='vkontakte_posts')
author_id = models.PositiveIntegerField(db_index=True)
author = generic.GenericForeignKey('author_content_type', 'author_id')
# abstract field for correct deleting group and user models in admin
group_wall = generic.GenericForeignKey('wall_owner_content_type',
'wall_owner_id')
user_wall = generic.GenericForeignKey('wall_owner_content_type',
'wall_owner_id')
group = generic.GenericForeignKey('author_content_type', 'author_id')
user = generic.GenericForeignKey('author_content_type', 'author_id')
date = models.DateTimeField(u'Время сообщения', db_index=True)
text = models.TextField(u'Текст записи')
comments = models.PositiveIntegerField(u'Кол-во комментариев',
default=0,
db_index=True)
likes = models.PositiveIntegerField(u'Кол-во лайков',
default=0,
db_index=True)
reposts = models.PositiveIntegerField(u'Кол-во репостов',
default=0,
db_index=True)
like_users = ManyToManyHistoryField(User, related_name='like_posts')
repost_users = ManyToManyHistoryField(User, related_name='repost_posts')
#{u'photo': {u'access_key': u'5f19dfdc36a1852824',
#u'aid': -7,
#u'created': 1333664090,
#u'height': 960,
#u'owner_id': 2462759,
#u'pid': 281543621,
#u'src': u'http://cs9733.userapi.com/u2462759/-14/m_fdad45ec.jpg',
#u'src_big': u'http://cs9733.userapi.com/u2462759/-14/x_60b1aed1.jpg',
#u'src_small': u'http://cs9733.userapi.com/u2462759/-14/s_d457021e.jpg',
#u'src_xbig': u'http://cs9733.userapi.com/u2462759/-14/y_b5a67b8d.jpg',
#u'src_xxbig': u'http://cs9733.userapi.com/u2462759/-14/z_5a64a153.jpg',
#u'text': u'',
#u'width': 1280},
#u'type': u'photo'}
#u'attachments': [{u'link': {u'description': u'',
#u'image_src': u'http://cs6030.userapi.com/u2462759/-2/x_cb9c00f8.jpg',
#u'title': u'SAAB_9000_CD_2_0_Turbo_190_k.jpg',
#u'url': u'http://www.yauto.cz/includes/img/inzerce/SAAB_9000_CD_2_0_Turbo_190_k.jpg'},
#u'type': u'link'}],
#attachments - содержит массив объектов, которые присоединены к текущей записи (фотографии, ссылки и т.п.). Более подробная информация представлена на странице Описание поля attachments
attachments = models.TextField()
media = models.TextField()
#{u'coordinates': u'55.6745689498 37.8724562529',
#u'place': {u'city': u'Moskovskaya oblast',
#u'country': u'Russian Federation',
#u'title': u'Shosseynaya ulitsa, Moskovskaya oblast'},
#u'type': u'point'}
#geo - если в записи содержится информация о местоположении, то она будет представлена в данном поле. Более подробная информация представлена на странице Описание поля geo
geo = models.TextField()
signer_id = models.PositiveIntegerField(
null=True,
help_text=
u'Eсли запись была опубликована от имени группы и подписана пользователем, то в поле содержится идентификатор её автора'
)
copy_owner_content_type = models.ForeignKey(
ContentType, related_name='vkontakte_wall_copy_posts', null=True)
copy_owner_id = models.PositiveIntegerField(
null=True,
db_index=True,
help_text=
u'Eсли запись является копией записи с чужой стены, то в поле содержится идентификатор владельца стены у которого была скопирована запись'
)
copy_owner = generic.GenericForeignKey('copy_owner_content_type',
'copy_owner_id')
# TODO: rename wall_reposts -> reposts, after renaming reposts -> reposts_count
copy_post = models.ForeignKey(
'Post',
null=True,
related_name='wall_reposts',
help_text=
u'Если запись является копией записи с чужой стены, то в поле содержится идентфикатор скопированной записи на стене ее владельца'
)
# copy_post_date = models.DateTimeField(u'Время сообщения-оригинала', null=True)
# copy_post_type = models.CharField(max_length=20)
copy_text = models.TextField(
u'Комментарий при репосте',
help_text=
u'Если запись является копией записи с чужой стены и при её копировании был добавлен комментарий, его текст содержится в данном поле'
)
# not in API
post_source = models.TextField()
online = models.PositiveSmallIntegerField(null=True)
reply_count = models.PositiveIntegerField(null=True)
objects = VkontakteCRUDManager()
remote = PostRemoteManager(remote_pk=('remote_id', ),
methods={
'get': 'get',
'getById': 'getById',
'create': 'post',
'update': 'edit',
'delete': 'delete',
'restore': 'restore',
})
@property
def reposters(self):
return [repost.author for repost in self.wall_reposts.all()]
def __unicode__(self):
return '%s: %s' % (unicode(self.wall_owner), self.text)
def save(self, *args, **kwargs):
# check strings for good encoding
# there is problems to save users with bad encoded activity strings like user ID=88798245
# try:
# self.text.encode('utf-16').decode('utf-16')
# except UnicodeDecodeError:
# self.text = ''
# поле назначено через API
if self.copy_owner_id and not self.copy_owner_content_type:
ct_model = User if self.copy_owner_id > 0 else Group
self.copy_owner_content_type = ContentType.objects.get_for_model(
ct_model)
self.copy_owner = ct_model.remote.fetch(
ids=[abs(self.copy_owner_id)])[0]
# save generic fields before saving post
if self.copy_owner:
self.copy_owner.save()
return super(Post, self).save(*args, **kwargs)
def prepare_create_params(self, **kwargs):
kwargs.update({
'owner_id': self.remote_owner_id,
'friends_only': kwargs.get('friends_only', 0),
'from_group': kwargs.get('from_group', ''),
'message': self.text,
'attachments': self.attachments,
'services': kwargs.get('services', ''),
'signed': 1 if self.signer_id else 0,
'publish_date': kwargs.get('publish_date', ''),
'lat': kwargs.get('lat', ''),
'long': kwargs.get('long', ''),
'place_id': kwargs.get('place_id', ''),
'post_id': kwargs.get('post_id', '')
})
return kwargs
def prepare_update_params(self, **kwargs):
return self.prepare_create_params(post_id=self.remote_id_short,
**kwargs)
def prepare_delete_params(self):
return {
'owner_id': self.remote_owner_id,
'post_id': self.remote_id_short
}
def parse_remote_id_from_response(self, response):
if response:
return '%s_%s' % (self.remote_owner_id, response['post_id'])
return None
def parse(self, response):
self.raw_json = dict(response)
for field_name in ['comments', 'likes', 'reposts']:
if field_name in response and 'count' in response[field_name]:
setattr(self, field_name, response.pop(field_name)['count'])
# TODO: may we should move this to save and keep parse queryless
self.wall_owner = self.get_or_create_group_or_user(
response.pop('to_id'))[0]
self.author = self.get_or_create_group_or_user(
response.pop('from_id'))[0]
response.pop('attachment', {})
for attachment in response.pop('attachments', []):
pass
# if attachment['type'] == 'poll':
# это можно делать только после сохранения поста, так что тольо через сигналы
# self.fetch_poll(attachment['poll']['poll_id'])
# TODO: this block broke tests with error
# IntegrityError: new row for relation "vkontakte_wall_post" violates check constraint "vkontakte_wall_post_copy_owner_id_check"
# if response.get('copy_owner_id'):
# try:
# self.copy_owner_content_type = ContentType.objects.get_for_model(User if response.get('copy_owner_id') > 0 else Group)
# self.copy_owner = self.copy_owner_content_type.get_object_for_this_type(remote_id=abs(response.get('copy_owner_id')))
# if response.get('copy_post_id'):
# self.copy_post = Post.objects.get(remote_id='%s_%s' % (response.get('copy_owner_id'), response.get('copy_post_id')))
# except ObjectDoesNotExist:
# pass
super(Post, self).parse(response)
self.remote_id = '%s%s_%s' % (
('-' if self.on_group_wall else ''), self.wall_owner.remote_id,
self.remote_id)
def fetch_comments(self, *args, **kwargs):
return Comment.remote.fetch_post(post=self, *args, **kwargs)
@transaction.commit_on_success
def fetch_likes(self, source='api', *args, **kwargs):
if source == 'api':
return super(Post, self).fetch_likes(*args, **kwargs)
else:
return self.fetch_likes_parser(*args, **kwargs)
@transaction.commit_on_success
def fetch_likes_parser(self, offset=0):
'''
Update and save fields:
* likes - count of likes
Update relations:
* like_users - users, who likes this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'likes/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 120
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 60
post_data['offset'] = offset
log.debug('Fetching likes of post "%s" of owner "%s", offset %d' %
(self.remote_id, self.wall_owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.likes = int(
parser.content_bs.find('a', {
'id': 'wk_likes_tablikes'
}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning(
'Strange markup of first page likes response: "%s"' %
parser.content)
self.like_users.clear()
#<div class="wk_likes_liker_row inl_bl" id="wk_likes_liker_row722246">
# <div class="wk_likes_likerph_wrap" onmouseover="WkView.likesBigphOver(this, 722246)">
# <a class="wk_likes_liker_ph" href="/kicolenka">
# <img class="wk_likes_liker_img" src="http://cs418825.vk.me/v418825246/6cf8/IBbSfmDz6R8.jpg" width="100" height="100" />
# </a>
# </div>
# <div class="wk_likes_liker_name"><a class="wk_likes_liker_lnk" href="/kicolenka">Оля Киселева</a></div>
#</div>
items = parser.add_users(
users=('div', {
'class': re.compile(r'^wk_likes_liker_row')
}),
user_link=('a', {
'class': 'wk_likes_liker_lnk'
}),
user_photo=('img', {
'class': 'wk_likes_liker_img'
}),
user_add=lambda user: self.like_users.add(user))
if len(items) == number_on_page:
self.fetch_likes_parser(offset=offset + number_on_page)
else:
return self.like_users.all()
def fetch_reposts(self, source='api', *args, **kwargs):
if source == 'api':
return self.fetch_reposts_api(*args, **kwargs)
else:
return self.fetch_reposts_parser(*args, **kwargs)
def fetch_reposts_api(self, *args, **kwargs):
self.fetch_instance_reposts(*args, **kwargs)
# update self.reposts
# commented, because it's less important count
# reposts_count = self.repost_users.get_query_set(only_pk=True).count()
# if reposts_count < self.reposts:
# log.warning('Fetched ammount of repost users less, than attribute `reposts` of post "%s": %d < %d' % (self.remote_id, reposts_count, self.reposts))
# self.reposts = reposts_count
# self.save()
return self.repost_users.all()
@transaction.commit_on_success
def fetch_instance_reposts(self, *args, **kwargs):
resources = self.fetch_repost_items(*args, **kwargs)
if not resources:
return Post.objects.none()
# TODO: still complicated to store reposts objects, may be it's task for another application
# posts = Post.remote.parse_response(resources)#, extra_fields={'copy_post_id': self.pk})
# return Post.objects.filter(pk__in=set([Post.remote.get_or_create_from_instance(instance).pk for instance in posts]))
# positive ids -> only users
# TODO: think about how to store reposts by groups
timestamps = dict([(post['from_id'], post['date'])
for post in resources if post['from_id'] > 0])
ids_new = timestamps.keys()
ids_current = self.repost_users.get_query_set(
only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)
ids_add = set(ids_new).difference(set(ids_current))
ids_remove = set(ids_current).difference(set(ids_new))
m2m_model = self.repost_users.through
# fetch new users
User.remote.fetch(ids=ids_add, only_expired=True)
# remove old reposts without time_from
self.repost_users.get_query_set_through().filter(
time_from=None).delete()
# add new reposts
get_repost_date = lambda id: datetime.fromtimestamp(timestamps[
id]) if id in timestamps else self.date
m2m_model.objects.bulk_create([
m2m_model(
**{
'user_id': id,
'post_id': self.pk,
'time_from': get_repost_date(id)
}) for id in ids_add
])
# remove reposts.
# Commented becouse of .using(MASTER_DATABASE).exclude(time_from=None) filtering for ids_current
# m2m_model.objects.filter(post_id=self.pk, user_id__in=ids_remove).update(time_to=datetime.now())
return
# не рекомендуется указывать default_count из-за бага паджинации репостов: https://vk.com/wall-51742963_6860
@fetch_all
def fetch_repost_items(self, offset=0, count=1000, *args, **kwargs):
if count > 1000:
raise ValueError("Parameter 'count' can not be more than 1000")
# owner_id
# идентификатор пользователя или сообщества, на стене которого находится запись. Если параметр не задан, то он считается равным идентификатору текущего пользователя.
# Обратите внимание, идентификатор сообщества в параметре owner_id необходимо указывать со знаком "-" — например, owner_id=-1 соответствует идентификатору сообщества ВКонтакте API (club1)
kwargs['owner_id'] = self.wall_owner.remote_id
if isinstance(self.wall_owner, Group):
kwargs['owner_id'] *= -1
# post_id
# идентификатор записи на стене.
kwargs['post_id'] = self.remote_id.split('_')[1]
# offset
# смещение, необходимое для выборки определенного подмножества записей.
kwargs['offset'] = int(offset)
# count
# количество записей, которое необходимо получить.
# положительное число, по умолчанию 20, максимальное значение 100
kwargs['count'] = int(count)
log.debug('Fetching repost users ids of post %s, offset %d' %
(self.remote_id, offset))
response = api_call('wall.getReposts', **kwargs)
return response['items']
@transaction.commit_on_success
def fetch_reposts_parser(self, offset=0):
'''
OLD method via parser, may works incorrect
Update and save fields:
* reposts - count of reposts
Update relations
* repost_users - users, who repost this post
'''
post_data = {
'act': 'show',
'al': 1,
'w': 'shares/wall%s' % self.remote_id,
}
if offset == 0:
number_on_page = 40
post_data['loc'] = 'wall%s' % self.remote_id,
else:
number_on_page = 20
post_data['offset'] = offset
log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' %
(self.remote_id, self.wall_owner, offset))
parser = VkontakteWallParser().request('/wkview.php', data=post_data)
if offset == 0:
try:
self.reposts = int(
parser.content_bs.find('a', {
'id': 'wk_likes_tabshares'
}).find('nobr').text.split()[0])
self.save()
except ValueError:
return
except:
log.warning(
'Strange markup of first page shares response: "%s"' %
parser.content)
self.repost_users.clear()
#<div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
# <div class="post_table">
# <div class="post_image">
# <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
# </div>
# <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">
items = parser.add_users(
users=('div', {
'id': re.compile('^post\d'),
'class': re.compile('^post ')
}),
user_link=('a', {
'class': 'author'
}),
user_photo=lambda item: item.find('a', {
'class': 'post_image'
}).find('img'),
user_add=lambda user: self.repost_users.add(user))
if len(items) == number_on_page:
self.fetch_reposts(offset=offset + number_on_page)
else:
return self.repost_users.all()
def fetch_statistic(self, *args, **kwargs):
if 'vkontakte_wall_statistic' not in settings.INSTALLED_APPS:
raise ImproperlyConfigured(
"Application 'vkontakte_wall_statistic' not in INSTALLED_APPS")
from vkontakte_wall_statistic.models import PostStatistic
return PostStatistic.remote.fetch(post=self, *args, **kwargs)
class Photo(PhotoBase):
class Meta:
verbose_name = u'Фотография Одноклассники'
verbose_name_plural = u'Фотографии Одноклассники'
remote_pk_field = 'id'
album = models.ForeignKey(Album, related_name='photos')
comments_count = models.PositiveIntegerField(default=0)
created = models.DateTimeField(null=True)
like_users = ManyToManyHistoryField(User, related_name='like_photos')
owner_content_type = models.ForeignKey(
ContentType, related_name='odnoklassniki_photos_owners')
owner_id = models.BigIntegerField(db_index=True)
owner = generic.GenericForeignKey('owner_content_type', 'owner_id')
pic1024max = models.URLField(null=True)
pic1024x768 = models.URLField(null=True)
pic128max = models.URLField(null=True)
pic128x128 = models.URLField(null=True)
pic180min = models.URLField(null=True)
pic190x190 = models.URLField(null=True)
pic240min = models.URLField(null=True)
pic320min = models.URLField(null=True)
pic50x50 = models.URLField(null=True)
pic640x480 = models.URLField(null=True)
standard_height = models.PositiveIntegerField(default=0)
standard_width = models.PositiveIntegerField(default=0)
text = models.TextField()
remote = PhotoRemoteManager(
methods={
'get': 'getPhotos',
'get_specific': 'getInfo',
'get_likes': 'getPhotoLikes',
})
def fetch_likes(self, **kwargs):
kwargs['photo_id'] = self.pk
return super(Photo, self).fetch_likes(**kwargs)
@property
def slug(self):
# Apparently there is no slug for a photo
return '%s' % (self.album.slug, )
def __unicode__(self):
return self.text
def parse(self, response):
created = response.pop('created_ms', None)
if created:
response[u'created'] = created / 1000
if response.get('album_id'):
self.album = Album.objects.get(id=int(response.get('album_id')))
return super(Photo, self).parse(response)