Exemple #1
0
class UserableModelMixin(models.Model):
    class Meta:
        abstract = True

    if 'vkontakte_users' in settings.INSTALLED_APPS:
        from vkontakte_users.models import User
        members = ManyToManyHistoryField(User,
                                         related_name='members_%(class)ss',
                                         versions=True)

        @atomic
        def update_members(self, *args, **kwargs):

            ids = self.__class__.remote.get_members_ids(group=self,
                                                        *args,
                                                        **kwargs)
            count = len(ids)
            initial = self.members.versions.count() == 0

            self.members = ids

            # update members_count
            if self.members_count != count:
                self.members_count = count
                self.save()

            if initial:
                self.members.get_query_set_through().update(time_from=None)
                self.members.versions.update(added_count=0)

            return self.members
    else:
        members = get_improperly_configured_field('vkontakte_users', True)
        update_members = get_improperly_configured_field('vkontakte_users')
class Status(TwitterBaseModel):

    author = models.ForeignKey('User', related_name='statuses')

    text = models.TextField()

    favorited = models.BooleanField(default=False)
    retweeted = models.BooleanField(default=False)
    truncated = models.BooleanField(default=False)

    source = models.CharField(max_length=100)
    source_url = models.URLField(null=True)

    favorites_count = models.PositiveIntegerField()
    retweets_count = models.PositiveIntegerField()
    replies_count = models.PositiveIntegerField(null=True)

    in_reply_to_status = models.ForeignKey('Status', null=True, related_name='replies')
    in_reply_to_user = models.ForeignKey('User', null=True, related_name='replies')

    favorites_users = ManyToManyHistoryField('User', related_name='favorites')
    retweeted_status = models.ForeignKey('Status', null=True, related_name='retweets')

    place = fields.JSONField(null=True)
    # format the next fields doesn't clear
    contributors = fields.JSONField(null=True)
    coordinates = fields.JSONField(null=True)
    geo = fields.JSONField(null=True)

    objects = models.Manager()
    remote = StatusManager(methods={
        'get': 'get_status',
    })

    def __unicode__(self):
        return u'%s: %s' % (self.author, self.text)

    @property
    def slug(self):
        return '/%s/status/%d' % (self.author.screen_name, self.id)

    def parse(self):
        self._response['favorites_count'] = self._response.pop('favorite_count', 0)
        self._response['retweets_count'] = self._response.pop('retweet_count', 0)

        self._response.pop('user', None)
        self._response.pop('in_reply_to_screen_name', None)
        self._response.pop('in_reply_to_user_id_str', None)
        self._response.pop('in_reply_to_status_id_str', None)

        self._get_foreignkeys_for_fields('in_reply_to_status', 'in_reply_to_user')

        super(Status, self).parse()

    def fetch_retweets(self, **kwargs):
        return Status.remote.fetch_retweets(status=self, **kwargs)

    def fetch_replies(self, **kwargs):
        return Status.remote.fetch_replies(status=self, **kwargs)
Exemple #3
0
class LikableModelMixin(models.Model):

    likes_users = ManyToManyHistoryField(User, related_name='like_%(class)ss')
    likes_count = models.PositiveIntegerField(u'Likes',
                                              null=True,
                                              db_index=True)

    class Meta:
        abstract = True

    @property
    def likes_remote_type(self):
        raise NotImplementedError()

    @atomic
    def fetch_likes(self, *args, **kwargs):

        kwargs['likes_type'] = self.likes_remote_type
        kwargs['item_id'] = self.remote_id_short
        kwargs['owner_id'] = self.owner_remote_id

        log.debug('Fetching likes of %s %s of owner "%s"' %
                  (self._meta.module_name, self.remote_id, self.owner))

        ids = User.remote.fetch_likes_user_ids(*args, **kwargs)
        self.likes_users = User.remote.fetch(ids=ids, only_expired=True)

        # update self.likes_count
        likes_count = self.likes_users.count()
        if likes_count < self.likes_count:
            log.warning(
                'Fetched ammount of like users less, than attribute `likes` of post "%s": %d < %d'
                % (self.remote_id, likes_count, self.likes_count))
        elif likes_count > self.likes_count:
            self.likes_count = likes_count
            self.save()

        return self.likes_users.all()

    def parse(self, response):
        if 'likes' in response:
            value = response.pop('likes')
            if isinstance(value, int):
                response['likes_count'] = value
            elif isinstance(value, dict) and 'count' in value:
                response['likes_count'] = value['count']
        super(LikableModelMixin, self).parse(response)
Exemple #4
0
class LikableModelMixin(models.Model):
    likes_users = ManyToManyHistoryField(User, related_name='like_%(class)ss')
    likes_count = models.PositiveIntegerField(null=True, help_text='The number of likes of this item')

    class Meta:
        abstract = True

    def parse(self, response):
        if 'like_count' in response:
            response['likes_count'] = response.pop('like_count')

        super(LikableModelMixin, self).parse(response)

    def update_count_and_get_like_users(self, instances, *args, **kwargs):
        self.likes_users = instances
        self.likes_count = instances.count()

        self.save()
        return instances


    # TODO: commented, becouse if many processes fetch_likes, got errors
    # DatabaseError: deadlock detected
    # DETAIL:  Process 27235 waits for ShareLock on transaction 3922627359; blocked by process 27037.
    # @atomic
    @fetch_all(return_all=update_count_and_get_like_users, paging_next_arg_name='after')
    def fetch_likes(self, limit=1000, **kwargs):
        """
        Retrieve and save all likes of post
        """
        ids = []
        response = api_call('%s/likes' % self.graph_id, limit=limit, **kwargs)
        if response:
            log.debug('response objects count=%s, limit=%s, after=%s' %
                      (len(response['data']), limit, kwargs.get('after')))
            for resource in response['data']:
                try:
                    user = get_or_create_from_small_resource(resource)
                    ids += [user.pk]
                except UnknownResourceType:
                    continue

        return User.objects.filter(pk__in=ids), response
Exemple #5
0
class Album(PhotoBase):
    class Meta:
        verbose_name = u'Альбом фотографий Одноклассники'
        verbose_name_plural = u'Альбомы фотографий Одноклассники'

    remote_pk_field = 'aid'

    created = models.DateField(null=True)

    like_users = ManyToManyHistoryField(User, related_name='like_albums')

    owner_content_type = models.ForeignKey(
        ContentType, related_name='odnoklassniki_albums_owners')
    owner_id = models.BigIntegerField(db_index=True)
    owner = generic.GenericForeignKey('owner_content_type', 'owner_id')

    photos_count = models.PositiveIntegerField(default=0)

    title = models.TextField()

    remote = AlbumRemoteManager(
        methods={
            'get': 'getAlbums',
            'get_one': 'getAlbumInfo',
            'get_likes': 'getAlbumLikes',
        })

    @property
    def slug(self):
        return '%s/album/%s' % (self.owner.slug, self.id)

    def __unicode__(self):
        return self.id

    def fetch_photos(self, **kwargs):
        return Photo.remote.fetch(group=self.owner, album=self, **kwargs)

    def fetch_likes(self, **kwargs):
        kwargs['aid'] = self.pk

        return super(Album, self).fetch_likes(**kwargs)
Exemple #6
0
class ReactionableModelMixin(models.Model):
    # without "Like": it may broke something
    reaction_types = ['love', 'wow', 'haha', 'sad', 'angry', 'thankful']

    reactions_count = models.PositiveIntegerField(null=True, help_text='The number of reactions of this item')

    def update_count_and_get_users_builder(reaction):

        def update_count_and_get_reaction_users(self, instances, *args, **kwargs):
            # setattr(self, '{0}s_count'.format(reaction), 0)
            setattr(self, '{0}s_users'.format(reaction), instances)
            setattr(self, '{0}s_count'.format(reaction), instances.count())

            self.save()
            return instances

        return update_count_and_get_reaction_users

    for reaction in reaction_types:
        related_name = '%s_' % reaction + '%(class)ss'
        vars()['{0}s_users'.format(reaction)] = ManyToManyHistoryField(User, related_name=related_name)
        vars()['{0}s_count'.format(reaction)] = models.PositiveIntegerField(null=True, help_text='The number of {0}s of this item'.format(reaction))

        vars()['update_count_and_get_{0}_users'.format(reaction)] = update_count_and_get_users_builder(reaction=reaction)


    class Meta:
        abstract = True

    def parse(self, response):
        for reaction in self.reaction_types:
            if '{0}_count'.format(reaction) in response:
                response['{0}s_count'.format(reaction)] = response.pop('{0}_count'.format(reaction))

        super(ReactionableModelMixin, self).parse(response)


    def fetch_reactions(self, reaction=None, limit=1000, **kwargs):
        """
        Retrieve and save all reactions of post

        Note: method may return different data structures:
            List:       if reaction is specified
            Dictionary: if reaction is not specified
        """
        ids = {}
        types = self.reaction_types + ['LIKE']
        for id_type in types:
            ids[id_type.upper()] = []

        response = api_call('%s/reactions' % self.graph_id, version=2.6, limit=limit, **kwargs)
        if response:
            log.debug('response objects count=%s, limit=%s, after=%s' %
                      (len(response['data']), limit, kwargs.get('after')))
            for resource in response['data']:
                try:
                    if (reaction != None) and (reaction.upper() != resource['type']):
                        continue
                    try:
                        user = get_or_create_from_small_resource(resource)
                        ids[resource['type']] += [user.pk]

                    except UnknownResourceType:
                        continue
                # no 'type' in resource
                except KeyError:
                    continue


        def get_user_ids(self, ids, response):
            return User.objects.filter(pk__in=ids), response

        result = {}
        for id_type in types:
            if (reaction != None) and (reaction.upper() != id_type.upper()):
                continue

            count_method = getattr(self, 'update_count_and_get_{0}_users'.format(id_type.lower()))
            # create count-and-get function wrapped in fetch_all decorator
            fetch = fetch_all(return_all=count_method, paging_next_arg_name='after')(get_user_ids)
            result[id_type.upper()] = fetch(self, ids[id_type.upper()], response)
            # for some reason fetch_all does not call count method
            count_method(result[id_type.upper()])

        if (reaction != None):
            return result[reaction.upper()]
        else:
            return result

    # separate from fetch method, because it would return wrong data if reaction specified
    def count_reactions(self):
        count = 0
        for reaction in self.reaction_types + ['like']:
            count += getattr(self, '{0}s_count'.format(reaction))

        self.reactions_count = count

        self.save()
Exemple #7
0
class User(TwitterBaseModel):

    screen_name = models.CharField(u'Screen name', max_length=50, unique=True)

    name = models.CharField(u'Name', max_length=100)
    description = models.TextField(u'Description')
    location = models.CharField(u'Location', max_length=100)
    time_zone = models.CharField(u'Time zone', max_length=100, null=True)

    contributors_enabled = models.BooleanField(u'Contributors enabled', default=False)
    default_profile = models.BooleanField(u'Default profile', default=False)
    default_profile_image = models.BooleanField(u'Default profile image', default=False)
    follow_request_sent = models.BooleanField(u'Follow request sent', default=False)
    following = models.BooleanField(u'Following', default=False)
    geo_enabled = models.BooleanField(u'Geo enabled', default=False)
    is_translator = models.BooleanField(u'Is translator', default=False)
    notifications = models.BooleanField(u'Notifications', default=False)
    profile_use_background_image = models.BooleanField(u'Profile use background image', default=False)
    protected = models.BooleanField(u'Protected', default=False)
    verified = models.BooleanField(u'Verified', default=False)

    profile_background_image_url = models.URLField(max_length=300, null=True)
    profile_background_image_url_https = models.URLField(max_length=300, null=True)
    profile_background_tile = models.BooleanField(default=False)
    profile_background_color = models.CharField(max_length=6)
    profile_banner_url = models.URLField(max_length=300, null=True)
    profile_image_url = models.URLField(max_length=300, null=True)
    profile_image_url_https = models.URLField(max_length=300)
    url = models.URLField(max_length=300, null=True)

    profile_link_color = models.CharField(max_length=6)
    profile_sidebar_border_color = models.CharField(max_length=6)
    profile_sidebar_fill_color = models.CharField(max_length=6)
    profile_text_color = models.CharField(max_length=6)

    favorites_count = models.PositiveIntegerField()
    followers_count = models.PositiveIntegerField()
    friends_count = models.PositiveIntegerField()
    listed_count = models.PositiveIntegerField()
    statuses_count = models.PositiveIntegerField()
    utc_offset = models.IntegerField(null=True)

    followers = ManyToManyHistoryField('User', versions=True)

    objects = models.Manager()
    remote = UserManager(methods={
        'get': 'get_user',
    })

    def __unicode__(self):
        return self.name

    def save(self, *args, **kwargs):
        if self.friends_count < 0:
            log.warning('Negative value friends_count=%s set to 0 for user ID %s' % (self.friends_count, self.id))
            self.friends_count = 0
        super(User, self).save(*args, **kwargs)

    @property
    def slug(self):
        return self.screen_name

    def parse(self):
        self._response['favorites_count'] = self._response.pop('favourites_count', None)
        self._response.pop('status', None)
        super(User, self).parse()

    def fetch_followers(self, **kwargs):
        return User.remote.fetch_followers_for_user(user=self, **kwargs)

    def get_followers_ids(self, **kwargs):
        return User.remote.get_followers_ids_for_user(user=self, **kwargs)

    def fetch_statuses(self, **kwargs):
        return Status.remote.fetch_for_user(user=self, **kwargs)
class ShareableModelMixin(models.Model):

    shares_users = ManyToManyHistoryField(User, related_name='shares_%(class)ss')
    shares_count = models.PositiveIntegerField(null=True, help_text='The number of shares of this item')

    class Meta:
        abstract = True

    def update_count_and_get_shares_users(self, instances, *args, **kwargs):
#        self.shares_users = instances
        # becouse here are not all shares: "Some posts may not appear here because of their privacy settings."
        if self.shares_count is None:
            self.shares_count = instances.count()
            self.save()
        return instances

    @atomic
    @fetch_all(return_all=update_count_and_get_shares_users, paging_next_arg_name='after')
    def fetch_shares(self, limit=1000, **kwargs):
        """
        Retrieve and save all shares of post
        """
        from facebook_api.models import MASTER_DATABASE  # here, becouse cycling import

        ids = []

        response = api_call('%s/sharedposts' % self.graph_id, **kwargs)
        if response:
            timestamps = dict(
                [(int(post['from']['id']), datetime_parse(post['created_time'])) for post in response['data']])
            ids_new = timestamps.keys()
            # becouse we should use local pk, instead of remote, remove it after pk -> graph_id
            ids_current = map(int, User.objects.filter(pk__in=self.shares_users.get_query_set(
                only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)).values_list('graph_id', flat=True))
            ids_add = set(ids_new).difference(set(ids_current))
            ids_add_pairs = []
            ids_remove = set(ids_current).difference(set(ids_new))

            log.debug('response objects count=%s, limit=%s, after=%s' %
                      (len(response['data']), limit, kwargs.get('after')))
            for post in response['data']:
                graph_id = int(post['from']['id'])
                if sorted(post['from'].keys()) == ['id', 'name']:
                    try:
                        user = get_or_create_from_small_resource(post['from'])
                        ids += [user.pk]
                        # this id in add list and still not in add_pairs (sometimes in response are duplicates)
                        if graph_id in ids_add and graph_id not in map(lambda i: i[0], ids_add_pairs):
                            # becouse we should use local pk, instead of remote
                            ids_add_pairs += [(graph_id, user.pk)]
                    except UnknownResourceType:
                        continue

            m2m_model = self.shares_users.through
            # '(album|post)_id'
            field_name = [f.attname for f in m2m_model._meta.local_fields
                          if isinstance(f, models.ForeignKey) and f.name != 'user'][0]

            # remove old shares without time_from
            self.shares_users.get_query_set_through().filter(time_from=None).delete()

            # in case some ids_add already left
            self.shares_users.get_query_set_through().filter(
                **{field_name: self.pk, 'user_id__in': map(lambda i: i[1], ids_add_pairs)}).delete()

            # add new shares with specified `time_from` value
            get_share_date = lambda id: timestamps[id] if id in timestamps else self.created_time
            m2m_model.objects.bulk_create([m2m_model(
                **{field_name: self.pk, 'user_id': pk, 'time_from': get_share_date(graph_id)}) for graph_id, pk in ids_add_pairs])

        return User.objects.filter(pk__in=ids), response
Exemple #9
0
class Media(InstagramBaseModel):
    remote_id = models.CharField(max_length=30, unique=True)
    caption = models.TextField(blank=True)
    link = models.URLField(max_length=68)

    type = models.CharField(max_length=5)
    filter = models.CharField(max_length=40)  # TODO: tune max_length of this field

    image_low_resolution = models.URLField(max_length=200)
    image_standard_resolution = models.URLField(max_length=200)
    image_thumbnail = models.URLField(max_length=200)

    video_low_bandwidth = models.URLField(max_length=130)
    video_low_resolution = models.URLField(max_length=130)
    video_standard_resolution = models.URLField(max_length=130)

    created_time = models.DateTimeField()

    comments_count = models.PositiveIntegerField(null=True)
    likes_count = models.PositiveIntegerField(null=True)

    user = models.ForeignKey('User', related_name="media_feed")
    location = models.ForeignKey('Location', null=True, related_name="media_feed")
    likes_users = ManyToManyHistoryField('User', related_name="likes_media")
    tags = models.ManyToManyField('Tag', related_name='media_feed')

    remote = MediaManager(remote_pk=('remote_id',), methods={
        'get': 'media',
        'user_recent_media': 'user_recent_media',
        'tag_recent_media': 'tag_recent_media',
        'location_recent_media': 'location_recent_media',
    })

    def get_url(self):
        return self.link

    def __unicode__(self):
        return self.caption

    def parse(self):
        self._response['remote_id'] = self._response.pop('id')

        for prefix in ['video', 'image']:
            key = '%ss' % prefix
            if key in self._response:
                for k, v in self._response[key].items():
                    media = self._response[key][k]
                    if isinstance(media, ApiModel):
                        media = media.__dict__
                    self._response['%s_%s' % (prefix, k)] = media['url']

        if not isinstance(self._response['created_time'], datetime):
            self._response['created_time'] = timestamp_to_datetime(self._response['created_time'])

        if 'comment_count' in self._response:
            self._response['comments_count'] = self._response.pop('comment_count')
        elif 'comments' in self._response:
            self._response['comments_count'] = self._response.pop('comments')['count']

        if 'like_count' in self._response:
            self._response['likes_count'] = self._response.pop('like_count')
        elif 'likes' in self._response:
            self._response['likes_count'] = self._response.pop('likes')['count']

        if isinstance(self._response['caption'], ApiModel):
            self._response['caption'] = self._response['caption'].text
        elif isinstance(self._response['caption'], dict):
            self._response['caption'] = self._response['caption']['text']

        # if 'likes' in self._response:
        #     self._response['likes_users'] = self._response.pop('likes')

        super(Media, self).parse()

    def fetch_comments(self):
        return Comment.remote.fetch_media_comments(self)

    def fetch_likes(self):
        return User.remote.fetch_media_likes(self)

    def save(self, *args, **kwargs):
        if self.caption is None:
            self.caption = ''

        super(Media, self).save(*args, **kwargs)

        for field, relations in self._relations_post_save['fk'].items():
            extra_fields = {'media_id': self.pk, 'owner_id': self.user_id} if field == 'comments' else {}
            for instance in relations:
                instance.__dict__.update(extra_fields)
                instance.__class__.remote.get_or_create_from_instance(instance)

        for field, relations in self._relations_post_save['m2m'].items():
            for instance in relations:
                instance = instance.__class__.remote.get_or_create_from_instance(instance)
                getattr(self, field).add(instance)
Exemple #10
0
class User(InstagramBaseModel):
    id = models.BigIntegerField(primary_key=True)
    username = models.CharField(max_length=30, unique=True)
    full_name = models.CharField(max_length=30)
    bio = models.CharField(max_length=150)

    profile_picture = models.URLField(max_length=112)
    website = models.URLField(max_length=150)  # found max_length=106

    followers_count = models.PositiveIntegerField(null=True, db_index=True)
    follows_count = models.PositiveIntegerField(null=True, db_index=True)
    media_count = models.PositiveIntegerField(null=True, db_index=True)

    followers = ManyToManyHistoryField('User', versions=True, related_name='follows')

    is_private = models.NullBooleanField('Account is private', db_index=True)

    objects = models.Manager()
    remote = UserManager(methods={
        'get': 'user',
        'search': 'user_search',
        'follows': 'user_follows',
        'followers': 'user_followed_by',
        'likes': 'media_likes',
    })

    @property
    def slug(self):
        return self.username

    def __unicode__(self):
        return self.full_name or self.username

    @property
    def instagram_link(self):
        return u'https://instagram.com/%s/' % self.username

    def _substitute(self, old_instance):
        super(User, self)._substitute(old_instance)
        for field_name in ['followers_count', 'follows_count', 'media_count', 'is_private']:
            if getattr(self, field_name) is None and getattr(old_instance, field_name) is not None:
                setattr(self, field_name, getattr(old_instance, field_name))

    def save(self, *args, **kwargs):

        # cut all CharFields to max allowed length
        for field in self._meta.local_fields:
            if isinstance(field, models.CharField):
                setattr(self, field.name, getattr(self, field.name)[:field.max_length])

        try:
            with atomic():
                super(InstagramModel, self).save(*args, **kwargs)
        except IntegrityError as e:
            if 'username' in e.message:
                # duplicate key value violates unique constraint "instagram_api_user_username_key"
                # DETAIL: Key (username)=(...) already exists.
                user_local = User.objects.get(username=self.username)
                try:
                    # check for recursive loop
                    # get remote user
                    user_remote = User.remote.get(user_local.pk)
                    try:
                        user_local2 = User.objects.get(username=user_remote.username)
                        # if users excahnge usernames or user is dead (400 error)
                        if user_local2.pk == self.pk or user_remote.is_private:
                            user_local.username = '******' % time.time()
                            user_local.save()
                    except User.DoesNotExist:
                        pass
                    # fetch right user
                    User.remote.fetch(user_local.pk)
                except InstagramError as e:
                    if e.code == 400:
                        user_local.delete()
                    else:
                        raise
                super(InstagramModel, self).save(*args, **kwargs)
            else:
                raise

    def parse(self):
        if isinstance(self._response, dict) and 'counts' in self._response:
            count = self._response['counts']
            self._response['followers_count'] = count.get('followed_by', 0)
            self._response['follows_count'] = count.get('follows', 0)
            self._response['media_count'] = count.get('media', 0)

        super(User, self).parse()

    def fetch_follows(self):
        return User.remote.fetch_follows(user=self)

    def fetch_followers(self):
        return User.remote.fetch_followers(user=self)

    def fetch_media(self, **kwargs):
        return Media.remote.fetch_user_media(user=self, **kwargs)

    def refresh(self):
        # do refresh via client_id, because is_private is dependent on access_token and relation with current user
        with override_api_context('instagram', use_client_id=True):
            super(User, self).refresh()
Exemple #11
0
class Media(InstagramBaseModel):
    remote_id = models.CharField(max_length=100, unique=True)
    caption = models.TextField(blank=True)
    link = models.URLField(max_length=300)

    type = models.CharField(max_length=20)

    image_low_resolution = models.URLField(max_length=200)
    image_standard_resolution = models.URLField(max_length=200)
    image_thumbnail = models.URLField(max_length=200)

    video_low_bandwidth = models.URLField(max_length=200)
    video_low_resolution = models.URLField(max_length=200)
    video_standard_resolution = models.URLField(max_length=200)

    created_time = models.DateTimeField()

    comments_count = models.PositiveIntegerField(null=True)
    likes_count = models.PositiveIntegerField(null=True)

    user = models.ForeignKey(User, related_name="media_feed")
    likes_users = ManyToManyHistoryField('User', related_name="likes_media")

    remote = MediaManager(remote_pk=('remote_id',), methods={
        'get': 'media',
        'user_recent_media': 'user_recent_media',
        'tag_recent_media': 'tag_recent_media',
    })

    def get_url(self):
        return self.link

    def __unicode__(self):
        return self.caption

    def parse(self):
        self._response['remote_id'] = self._response.pop('id')

        for prefix in ['video', 'image']:
            key = '%ss' % prefix
            if key in self._response:
                for k, v in self._response[key].items():
                    media = self._response[key][k]
                    if isinstance(media, ApiModel):
                        media = media.__dict__
                    self._response['%s_%s' % (prefix, k)] = media['url']

        if not isinstance(self._response['created_time'], datetime):
            self._response['created_time'] = timestamp_to_datetime(self._response['created_time'])

        if 'comment_count' in self._response:
            self._response['comments_count'] = self._response.pop('comment_count')
        elif 'comments' in self._response:
            self._response['comments_count'] = self._response.pop('comments')['count']

        if 'like_count' in self._response:
            self._response['likes_count'] = self._response.pop('like_count')
        elif 'likes' in self._response:
            self._response['likes_count'] = self._response.pop('likes')['count']

        if isinstance(self._response['caption'], ApiModel):
            self._response['caption'] = self._response['caption'].text
        elif isinstance(self._response['caption'], dict):
            self._response['caption'] = self._response['caption']['text']

        super(Media, self).parse()

    def fetch_comments(self):
        return Comment.remote.fetch_media_comments(self)

    def fetch_likes(self):
        return User.remote.fetch_media_likes(self)

    def save(self, *args, **kwargs):
        if self.caption is None:
            self.caption = ''
        super(Media, self).save(*args, **kwargs)
Exemple #12
0
class RepostableModelMixin(models.Model):

    reposts_users = ManyToManyHistoryField(User,
                                           related_name='reposts_%(class)ss')
    reposts_count = models.PositiveIntegerField(u'Кол-во репостов',
                                                null=True,
                                                db_index=True)

    class Meta:
        abstract = True

    def parse(self, response):
        if 'reposts' in response:
            value = response.pop('reposts')
            if isinstance(value, int):
                response['reposts_count'] = value
            elif isinstance(value, dict) and 'count' in value:
                response['reposts_count'] = value['count']
        super(RepostableModelMixin, self).parse(response)

    @property
    def reposters(self):
        return [repost.author for repost in self.wall_reposts.all()]

    def fetch_reposts(self, source='api', *args, **kwargs):
        if source == 'api':
            return self.fetch_reposts_api(*args, **kwargs)
        else:
            return self.fetch_reposts_parser(*args, **kwargs)

    def fetch_reposts_api(self, *args, **kwargs):
        self.fetch_instance_reposts(*args, **kwargs)

        # update self.reposts_count
        reposts_count = self.reposts_users.get_query_set(only_pk=True).count()
        if reposts_count < self.reposts_count:
            log.warning(
                'Fetched ammount of repost users less, than attribute `reposts` of post "%s": %d < %d'
                % (self.remote_id, reposts_count, self.reposts_count))
        elif reposts_count > self.reposts_count:
            self.reposts_count = reposts_count
            self.save()

        return self.reposts_users.all()

    @atomic
    def fetch_instance_reposts(self, *args, **kwargs):
        resources = self.fetch_reposts_items(*args, **kwargs)
        if not resources:
            return self.__class__.objects.none()

        # TODO: still complicated to store reposts objects, may be it's task for another application
#         posts = Post.remote.parse_response(resources, extra_fields={'copy_post_id': self.pk})
#         Post.objects.filter(pk__in=set([Post.remote.get_or_create_from_instance(instance).pk
#         for instance in posts]))

# positive ids -> only users
# TODO: think about how to store reposts by groups
        timestamps = dict([(post['from_id'], post['date'])
                           for post in resources if post['from_id'] > 0])
        ids_new = timestamps.keys()
        ids_current = self.reposts_users.get_query_set(
            only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)
        ids_current_left = self.reposts_users.get_query_set_through().using(MASTER_DATABASE).exclude(time_to=None) \
            .values_list('user_id', flat=True)
        ids_add = set(ids_new).difference(set(ids_current))
        ids_remove = set(ids_current).difference(set(ids_new))
        # some of them may be already left for some reason or API error
        ids_unleft = set(ids_add).intersection(set(ids_current_left))
        ids_add = ids_add.difference(ids_unleft)

        # fetch new users
        User.remote.fetch(ids=ids_add, only_expired=True)

        # remove old reposts without time_from
        self.reposts_users.get_query_set_through().filter(
            time_from=None).delete()

        # try to find left users, that present in ids_add and make them unleft
        self.reposts_users.get_query_set_through().exclude(
            time_to=None).filter(user_id__in=ids_unleft).update(time_to=None)

        # add new reposts
        get_repost_date = lambda id: datetime.utcfromtimestamp(timestamps[
            id]).replace(tzinfo=timezone.utc
                         ) if id in timestamps else self.date

        m2m_model = self.reposts_users.through
        m2m_model.objects.bulk_create([
            m2m_model(
                **{
                    'user_id': id,
                    'post_id': self.pk,
                    'time_from': get_repost_date(id)
                }) for id in ids_add
        ])

        # remove reposts
        self.reposts_users.get_query_set_through().filter(
            user_id__in=ids_remove).update(time_to=timezone.now())

    # не рекомендуется указывать default_count из-за бага паджинации репостов: https://vk.com/wall-51742963_6860
    @fetch_all(max_extra_calls=3)
    def fetch_reposts_items(self, offset=0, count=1000, *args, **kwargs):
        if count > 1000:
            raise ValueError("Parameter 'count' can not be more than 1000")

        # owner_id
        # идентификатор пользователя или сообщества, на стене которого находится запись. Если параметр не задан, то он считается равным идентификатору текущего пользователя.
        # Обратите внимание, идентификатор сообщества в параметре owner_id необходимо указывать со знаком "-" — например, owner_id=-1 соответствует идентификатору сообщества ВКонтакте API (club1)
        kwargs['owner_id'] = self.owner_remote_id
        # post_id
        # идентификатор записи на стене.
        kwargs['post_id'] = self.remote_id_short
        # offset
        # смещение, необходимое для выборки определенного подмножества записей.
        kwargs['offset'] = int(offset)
        # count
        # количество записей, которое необходимо получить.
        # положительное число, по умолчанию 20, максимальное значение 100
        kwargs['count'] = int(count)

        response = api_call('wall.getReposts', **kwargs)
        log.debug(
            'Fetching reposts for post %s: %d returned, offset %d, count %d' %
            (self.remote_id, len(response['items']), offset, count))
        return response['items']

    @atomic
    def fetch_reposts_parser(self, offset=0):
        '''
        OLD method via parser, may works incorrect
        Update and save fields:
            * reposts - count of reposts
        Update relations
            * reposts_users - users, who repost this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'shares/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 40
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 20
            post_data['offset'] = offset

        log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' %
                  (self.remote_id, self.owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)
        if offset == 0:
            try:
                self.reposts_count = int(
                    parser.content_bs.find('a', {
                        'id': 'wk_likes_tabshares'
                    }).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning(
                    'Strange markup of first page shares response: "%s"' %
                    parser.content)
            self.reposts_users.clear()

        # <div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
        #  <div class="post_table">
        #    <div class="post_image">
        #      <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
        #    </div>
        #      <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">

        items = parser.add_users(
            users=('div', {
                'id': re.compile('^post\d'),
                'class': re.compile('^post ')
            }),
            user_link=('a', {
                'class': 'author'
            }),
            user_photo=lambda item: item.find('a', {
                'class': 'post_image'
            }).find('img'),
            user_add=lambda user: self.reposts_users.add(user))

        if len(items) == number_on_page:
            self.fetch_reposts(offset=offset + number_on_page)
        else:
            return self.reposts_users.all()
Exemple #13
0
class Comment(OdnoklassnikiModel):

    methods_namespace = 'discussions'

    # temporary variable for distance from parse() to save()
    author_type = None

    id = models.CharField(max_length=68, primary_key=True)

    discussion = models.ForeignKey(Discussion, related_name='comments')

    # denormalization for query optimization
    owner_content_type = models.ForeignKey(
        ContentType, related_name='odnoklassniki_comments_owners')
    owner_id = models.BigIntegerField(db_index=True)
    owner = generic.GenericForeignKey('owner_content_type', 'owner_id')

    author_content_type = models.ForeignKey(
        ContentType, related_name='odnoklassniki_comments_authors')
    author_id = models.BigIntegerField(db_index=True)
    author = generic.GenericForeignKey('author_content_type', 'author_id')

    reply_to_comment = models.ForeignKey(
        'self', null=True, verbose_name=u'Это ответ на комментарий')

    reply_to_author_content_type = models.ForeignKey(
        ContentType,
        null=True,
        related_name='odnoklassniki_comments_reply_to_authors')
    reply_to_author_id = models.BigIntegerField(db_index=True, null=True)
    reply_to_author = generic.GenericForeignKey('reply_to_author_content_type',
                                                'reply_to_author_id')

    object_type = models.CharField(max_length=20, choices=COMMENT_TYPE_CHOICES)
    text = models.TextField()

    date = models.DateTimeField()

    likes_count = models.PositiveIntegerField(default=0)
    liked_it = models.BooleanField(default=False)

    attrs = JSONField(null=True)

    like_users = ManyToManyHistoryField(User, related_name='like_comments')

    remote = CommentRemoteManager(
        methods={
            'get': 'getComments',
            'get_one': 'getComment',
            'get_likes': 'getCommentLikes',
        })

    class Meta:
        verbose_name = _('Odnoklassniki comment')
        verbose_name_plural = _('Odnoklassniki comments')

    @property
    def slug(self):
        return self.discussion.slug

    def save(self, *args, **kwargs):
        self.owner = self.discussion.owner

        if self.author_id and not self.author:
            if self.author_type == 'GROUP':
                if self.author_id == self.owner_id:
                    self.author = self.owner
                else:
                    from odnoklassniki_groups.models import Group
                    try:
                        self.author = Group.remote.fetch(
                            ids=[self.author_id])[0]
                    except IndexError:
                        raise Exception(
                            "Can't fetch Odnoklassniki comment's group-author with ID %s"
                            % self.author_id)
            else:
                try:
                    self.author = User.objects.get(pk=self.author_id)
                except User.DoesNotExist:
                    try:
                        self.author = User.remote.fetch(
                            ids=[self.author_id])[0]
                    except IndexError:
                        raise Exception(
                            "Can't fetch Odnoklassniki comment's user-author with ID %s"
                            % self.author_id)

        # it's hard to get proper reply_to_author_content_type in case we fetch comments from last
        if self.reply_to_author_id and not self.reply_to_author_content_type:
            self.reply_to_author_content_type = ContentType.objects.get_for_model(
                User)
#         if self.reply_to_comment_id and self.reply_to_author_id and not self.reply_to_author_content_type:
#             try:
#                 self.reply_to_author = User.objects.get(pk=self.reply_to_author_id)
#             except User.DoesNotExist:
#                 self.reply_to_author = self.reply_to_comment.author

# check for existing comment from self.reply_to_comment to prevent ItegrityError
        if self.reply_to_comment_id:
            try:
                self.reply_to_comment = Comment.objects.get(
                    pk=self.reply_to_comment_id)
            except Comment.DoesNotExist:
                log.error(
                    "Try to save comment ID=%s with reply_to_comment_id=%s that doesn't exist in DB"
                    % (self.id, self.reply_to_comment_id))
                self.reply_to_comment = None

        return super(Comment, self).save(*args, **kwargs)

    def parse(self, response):
        # rename becouse discussion has object_type
        if 'type' in response:
            response['object_type'] = response.pop('type')

        if 'like_count' in response:
            response['likes_count'] = response.pop('like_count')
        if 'reply_to_id' in response:
            response['reply_to_author_id'] = response.pop('reply_to_id')
        if 'reply_to_comment_id' in response:
            response['reply_to_comment'] = response.pop('reply_to_comment_id')

        # if author is a group
        if 'author_type' in response:
            response.pop('author_name')
            self.author_type = response.pop('author_type')

        return super(Comment, self).parse(response)

    def update_likes_count(self, instances, *args, **kwargs):
        users = User.objects.filter(pk__in=instances)
        self.like_users = users
        self.likes_count = len(instances)
        self.save()
        return users

    @atomic
    @fetch_all(return_all=update_likes_count, has_more=None)
    def fetch_likes(self, count=100, **kwargs):
        kwargs['comment_id'] = self.id
        kwargs['discussionId'] = self.discussion.id
        kwargs['discussionType'] = self.discussion.object_type
        kwargs['count'] = int(count)
        #        kwargs['fields'] = Comment.remote.get_request_fields('user')

        response = Comment.remote.api_call(method='get_likes', **kwargs)
        # has_more not in dict and we need to handle pagination manualy
        if 'users' not in response:
            response.pop('anchor', None)
            users_ids = []
        else:
            users_ids = list(
                User.remote.get_or_create_from_resources_list(
                    response['users']).values_list('pk', flat=True))

        return users_ids, response
Exemple #14
0
class Discussion(OdnoklassnikiPKModel):

    methods_namespace = ''
    remote_pk_field = 'object_id'

    owner_content_type = models.ForeignKey(
        ContentType, related_name='odnoklassniki_discussions_owners')
    owner_id = models.BigIntegerField(db_index=True)
    owner = generic.GenericForeignKey('owner_content_type', 'owner_id')

    author_content_type = models.ForeignKey(
        ContentType, related_name='odnoklassniki_discussions_authors')
    author_id = models.BigIntegerField(db_index=True)
    author = generic.GenericForeignKey('author_content_type', 'author_id')

    object_type = models.CharField(max_length=20,
                                   choices=DISCUSSION_TYPE_CHOICES,
                                   default=DISCUSSION_TYPE_DEFAULT)
    title = models.TextField()
    message = models.TextField()

    date = models.DateTimeField(db_index=True)
    last_activity_date = models.DateTimeField(null=True)
    last_user_access_date = models.DateTimeField(null=True)

    new_comments_count = models.PositiveIntegerField(default=0)
    comments_count = models.PositiveIntegerField(default=0)
    likes_count = models.PositiveIntegerField(default=0)
    reshares_count = models.PositiveIntegerField(default=0)

    # vote
    last_vote_date = models.DateTimeField(null=True)
    votes_count = models.PositiveIntegerField(default=0)
    question = models.TextField()

    liked_it = models.BooleanField(default=False)

    entities = JSONField(null=True)
    ref_objects = JSONField(null=True)
    attrs = JSONField(null=True)

    like_users = ManyToManyHistoryField(User, related_name='like_discussions')

    remote = DiscussionRemoteManager(
        methods={
            'get': 'discussions.getList',
            'get_one': 'discussions.get',
            'get_likes': 'discussions.getDiscussionLikes',
            'stream': 'stream.get',
            'mget': 'mediatopic.getByIds',
        })

    #     def __unicode__(self):
    #         return self.name

    class Meta:
        verbose_name = _('Odnoklassniki discussion')
        verbose_name_plural = _('Odnoklassniki discussions')

    def _substitute(self, old_instance):
        super(Discussion, self)._substitute(old_instance)
        try:
            if self.entities['themes'][0]['images'][0] is None:
                self.entities['themes'][0]['images'][
                    0] = old_instance.entities['themes'][0]['images'][0]
        except (KeyError, TypeError):
            pass

    def save(self, *args, **kwargs):

        # make 2 dicts {id: instance} for group and users from entities
        if self.entities:
            entities = {
                'users': [],
                'groups': [],
            }
            for resource in self.entities.get('users', []):
                entities['users'] += [
                    User.remote.get_or_create_from_resource(resource)
                ]
            for resource in self.entities.get('groups', []):
                from odnoklassniki_groups.models import Group
                entities['groups'] += [
                    Group.remote.get_or_create_from_resource(resource)
                ]
            for field in ['users', 'groups']:
                entities[field] = dict([(instance.id, instance)
                                        for instance in entities[field]])

            # set owner
            if self.ref_objects:
                for resource in self.ref_objects:
                    id = int(resource['id'])
                    if resource['type'] == 'GROUP':
                        self.owner = entities['groups'][id]
                    elif resource['type'] == 'USER':
                        self.owner = entities['user'][id]
                    else:
                        log.warning(
                            "Strange type of object in ref_objects %s for duscussion ID=%s"
                            % (resource, self.id))

            # set author
            if self.author_id:
                if self.author_id in entities['groups']:
                    self.author = entities['groups'][self.author_id]
                elif self.author_id in entities['users']:
                    self.author = entities['users'][self.author_id]
                else:
                    log.warning(
                        "Imposible to find author with ID=%s in entities of duscussion ID=%s"
                        % (self.author_id, self.id))
                    self.author_id = None

        if self.owner and not self.author_id:
            # of no author_id (owner_uid), so it's equal to owner from ref_objects
            self.author = self.owner

        if self.author_id and not self.author:
            self.author = self.author_content_type.model_class(
            ).objects.get_or_create(pk=self.author_id)[0]

        if self.owner_id and not self.owner:
            self.owner = self.owner_content_type.model_class(
            ).objects.get_or_create(pk=self.owner_id)[0]

        return super(Discussion, self).save(*args, **kwargs)

    @property
    def refresh_kwargs(self):
        return {
            'id': self.id,
            'type': self.object_type or DISCUSSION_TYPE_DEFAULT
        }

    @property
    def slug(self):
        return '%s/topic/%s' % (self.owner.slug, self.id)

    def parse(self, response):
        from odnoklassniki_groups.models import Group
        if 'discussion' in response:
            response.update(response.pop('discussion'))

        # Discussion.remote.fetch_one
        if 'entities' in response and 'media_topics' in response['entities'] \
            and len(response['entities']['media_topics']) == 1:
            response.update(response['entities'].pop('media_topics')[0])
            if 'polls' in response['entities']:
                response.update(response['entities'].pop('polls')[0])
                if 'vote_summary' in response:
                    response['last_vote_date'] = response['vote_summary'][
                        'last_vote_date_ms'] / 1000
                    response['votes_count'] = response['vote_summary']['count']

        # media_topics
        if 'like_summary' in response:
            response['likes_count'] = response['like_summary']['count']
            response.pop('like_summary')
        if 'reshare_summary' in response:
            response['reshares_count'] = response['reshare_summary']['count']
            response.pop('reshare_summary')
        if 'discussion_summary' in response:
            response['comments_count'] = response['discussion_summary'][
                'comments_count']
            response.pop('discussion_summary')
        if 'author_ref' in response:
            i = response.pop('author_ref').split(':')
            response['author_id'] = i[1]
            self.author_content_type = ContentType.objects.get(
                app_label='odnoklassniki_%ss' % i[0], model=i[0])
        if 'owner_ref' in response:
            i = response.pop('owner_ref').split(':')
            response['owner_id'] = i[1]
            self.owner_content_type = ContentType.objects.get(
                app_label='odnoklassniki_%ss' % i[0], model=i[0])
        if 'created_ms' in response:
            response['date'] = response.pop('created_ms') / 1000
        if 'media' in response:
            response['title'] = response['media'][0]['text']

        # in API owner is author
        if 'owner_uid' in response:
            response['author_id'] = response.pop('owner_uid')

        # some name cleaning
        if 'like_count' in response:
            response['likes_count'] = response.pop('like_count')

        if 'total_comments_count' in response:
            response['comments_count'] = response.pop('total_comments_count')

        if 'creation_date' in response:
            response['date'] = response.pop('creation_date')

        # response of stream.get has another format
        if 'message' in response and '{media_topic' in response['message']:
            regexp = r'{media_topic:?(\d+)?}'
            m = re.findall(regexp, response['message'])
            if len(m):
                response['id'] = m[0]
                response['message'] = re.sub(regexp, '', response['message'])

        return super(Discussion, self).parse(response)

    def fetch_comments(self, **kwargs):
        return Comment.remote.fetch(discussion=self, **kwargs)

    def update_likes_count(self, instances, *args, **kwargs):
        users = User.objects.filter(pk__in=instances)
        self.like_users = users
        self.likes_count = len(instances)
        self.save()
        return users

    @atomic
    @fetch_all(return_all=update_likes_count, has_more=None)
    def fetch_likes(self, count=100, **kwargs):
        kwargs['discussionId'] = self.id
        kwargs['discussionType'] = self.object_type
        kwargs['count'] = int(count)
        #        kwargs['fields'] = Discussion.remote.get_request_fields('user')

        response = Discussion.remote.api_call(method='get_likes', **kwargs)
        # has_more not in dict and we need to handle pagination manualy
        if 'users' not in response:
            response.pop('anchor', None)
            users_ids = []
        else:
            users_ids = list(
                User.remote.get_or_create_from_resources_list(
                    response['users']).values_list('pk', flat=True))

        return users_ids, response
Exemple #15
0
class Comment(WallAbstractModel):
    class Meta:
        verbose_name = u'Коментарий сообщения Вконтакте'
        verbose_name_plural = u'Комментарии сообщений Вконтакте'

    remote_pk_field = 'cid'
    likes_type = 'comment'
    fields_required_for_update = ['comment_id', 'post_id', 'owner_id']

    post = models.ForeignKey(Post,
                             verbose_name=u'Пост',
                             related_name='wall_comments')

    # Владелец стены сообщения User or Group (декомпозиция от self.post для фильтра в админке и быстрых запросов)
    wall_owner_content_type = models.ForeignKey(
        ContentType, related_name='vkontakte_wall_comments')
    wall_owner_id = models.PositiveIntegerField(db_index=True)
    wall_owner = generic.GenericForeignKey('wall_owner_content_type',
                                           'wall_owner_id')

    # Автор комментария
    author_content_type = models.ForeignKey(ContentType,
                                            related_name='comments')
    author_id = models.PositiveIntegerField(db_index=True)
    author = generic.GenericForeignKey('author_content_type', 'author_id')

    from_id = models.IntegerField(
        null=True)  # strange value, seems to be equal to author

    # Это ответ пользователю
    reply_for_content_type = models.ForeignKey(ContentType,
                                               null=True,
                                               related_name='replies')
    reply_for_id = models.PositiveIntegerField(null=True, db_index=True)
    reply_for = generic.GenericForeignKey('reply_for_content_type',
                                          'reply_for_id')

    reply_to = models.ForeignKey('self',
                                 null=True,
                                 verbose_name=u'Это ответ на комментарий')

    # abstract field for correct deleting group and user models in admin
    group = generic.GenericForeignKey('author_content_type', 'author_id')
    user = generic.GenericForeignKey('author_content_type', 'author_id')
    group_wall_reply = generic.GenericForeignKey('reply_for_content_type',
                                                 'reply_for_id')
    user_wall_reply = generic.GenericForeignKey('reply_for_content_type',
                                                'reply_for_id')

    date = models.DateTimeField(u'Время комментария', db_index=True)
    text = models.TextField(u'Текст комментария')

    likes = models.PositiveIntegerField(u'Кол-во лайков',
                                        default=0,
                                        db_index=True)

    like_users = ManyToManyHistoryField(User, related_name='like_comments')

    objects = VkontakteCRUDManager()
    remote = CommentRemoteManager(remote_pk=('remote_id', ),
                                  methods={
                                      'get': 'getComments',
                                      'create': 'addComment',
                                      'update': 'editComment',
                                      'delete': 'deleteComment',
                                      'restore': 'restoreComment',
                                  })

    def save(self, *args, **kwargs):
        self.wall_owner = self.post.wall_owner
        return super(Comment, self).save(*args, **kwargs)

    def prepare_create_params(self, **kwargs):
        kwargs.update({
            'owner_id':
            self.remote_owner_id,
            'post_id':
            self.post.remote_id_short,
            'text':
            self.text,
            'reply_to_comment':
            self.reply_for.id if self.reply_for else '',
            'from_group':
            int(kwargs.get('from_group', 0)),
            'attachments':
            kwargs.get('attachments', ''),
        })
        return kwargs

    def prepare_update_params(self, **kwargs):
        kwargs.update({
            'owner_id': self.remote_owner_id,
            'comment_id': self.remote_id_short,
            'message': self.text,
            'attachments': kwargs.get('attachments', ''),
        })
        return kwargs

    def prepare_delete_params(self):
        return {
            'owner_id': self.remote_owner_id,
            'comment_id': self.remote_id_short
        }

    def parse_remote_id_from_response(self, response):
        if response:
            return '%s_%s' % (self.remote_owner_id, response['cid'])
        return None

    def parse(self, response):
        self.raw_json = response
        super(Comment, self).parse(response)

        if '_' not in str(self.remote_id):
            self.remote_id = '%s_%s' % (self.post.remote_id.split('_')[0],
                                        self.remote_id)

        for field_name in ['likes']:
            if field_name in response and 'count' in response[field_name]:
                setattr(self, field_name, response.pop(field_name)['count'])

        self.author = User.objects.get_or_create(remote_id=response['uid'])[0]

        if 'reply_to_uid' in response:
            self.reply_for = User.objects.get_or_create(
                remote_id=response['reply_to_uid'])[0]
        if 'reply_to_cid' in response:
            try:
                self.reply_to = Comment.objects.get(
                    remote_id=response['reply_to_cid'])
            except:
                pass
Exemple #16
0
class Post(WallAbstractModel):
    class Meta:
        verbose_name = u'Сообщение Вконтакте'
        verbose_name_plural = u'Сообщения Вконтакте'

    likes_type = 'post'
    fields_required_for_update = ['post_id', 'owner_id']

    # Владелец стены сообщения User or Group
    wall_owner_content_type = models.ForeignKey(
        ContentType, related_name='vkontakte_wall_posts')
    wall_owner_id = models.PositiveIntegerField(db_index=True)
    wall_owner = generic.GenericForeignKey('wall_owner_content_type',
                                           'wall_owner_id')

    # Создатель/автор сообщения
    author_content_type = models.ForeignKey(ContentType,
                                            related_name='vkontakte_posts')
    author_id = models.PositiveIntegerField(db_index=True)
    author = generic.GenericForeignKey('author_content_type', 'author_id')

    # abstract field for correct deleting group and user models in admin
    group_wall = generic.GenericForeignKey('wall_owner_content_type',
                                           'wall_owner_id')
    user_wall = generic.GenericForeignKey('wall_owner_content_type',
                                          'wall_owner_id')
    group = generic.GenericForeignKey('author_content_type', 'author_id')
    user = generic.GenericForeignKey('author_content_type', 'author_id')

    date = models.DateTimeField(u'Время сообщения', db_index=True)
    text = models.TextField(u'Текст записи')

    comments = models.PositiveIntegerField(u'Кол-во комментариев',
                                           default=0,
                                           db_index=True)
    likes = models.PositiveIntegerField(u'Кол-во лайков',
                                        default=0,
                                        db_index=True)
    reposts = models.PositiveIntegerField(u'Кол-во репостов',
                                          default=0,
                                          db_index=True)

    like_users = ManyToManyHistoryField(User, related_name='like_posts')
    repost_users = ManyToManyHistoryField(User, related_name='repost_posts')

    #{u'photo': {u'access_key': u'5f19dfdc36a1852824',
    #u'aid': -7,
    #u'created': 1333664090,
    #u'height': 960,
    #u'owner_id': 2462759,
    #u'pid': 281543621,
    #u'src': u'http://cs9733.userapi.com/u2462759/-14/m_fdad45ec.jpg',
    #u'src_big': u'http://cs9733.userapi.com/u2462759/-14/x_60b1aed1.jpg',
    #u'src_small': u'http://cs9733.userapi.com/u2462759/-14/s_d457021e.jpg',
    #u'src_xbig': u'http://cs9733.userapi.com/u2462759/-14/y_b5a67b8d.jpg',
    #u'src_xxbig': u'http://cs9733.userapi.com/u2462759/-14/z_5a64a153.jpg',
    #u'text': u'',
    #u'width': 1280},
    #u'type': u'photo'}

    #u'attachments': [{u'link': {u'description': u'',
    #u'image_src': u'http://cs6030.userapi.com/u2462759/-2/x_cb9c00f8.jpg',
    #u'title': u'SAAB_9000_CD_2_0_Turbo_190_k.jpg',
    #u'url': u'http://www.yauto.cz/includes/img/inzerce/SAAB_9000_CD_2_0_Turbo_190_k.jpg'},
    #u'type': u'link'}],
    #attachments - содержит массив объектов, которые присоединены к текущей записи (фотографии, ссылки и т.п.). Более подробная информация представлена на странице Описание поля attachments
    attachments = models.TextField()
    media = models.TextField()

    #{u'coordinates': u'55.6745689498 37.8724562529',
    #u'place': {u'city': u'Moskovskaya oblast',
    #u'country': u'Russian Federation',
    #u'title': u'Shosseynaya ulitsa, Moskovskaya oblast'},
    #u'type': u'point'}
    #geo - если в записи содержится информация о местоположении, то она будет представлена в данном поле. Более подробная информация представлена на странице Описание поля geo
    geo = models.TextField()

    signer_id = models.PositiveIntegerField(
        null=True,
        help_text=
        u'Eсли запись была опубликована от имени группы и подписана пользователем, то в поле содержится идентификатор её автора'
    )

    copy_owner_content_type = models.ForeignKey(
        ContentType, related_name='vkontakte_wall_copy_posts', null=True)
    copy_owner_id = models.PositiveIntegerField(
        null=True,
        db_index=True,
        help_text=
        u'Eсли запись является копией записи с чужой стены, то в поле содержится идентификатор владельца стены у которого была скопирована запись'
    )
    copy_owner = generic.GenericForeignKey('copy_owner_content_type',
                                           'copy_owner_id')

    # TODO: rename wall_reposts -> reposts, after renaming reposts -> reposts_count
    copy_post = models.ForeignKey(
        'Post',
        null=True,
        related_name='wall_reposts',
        help_text=
        u'Если запись является копией записи с чужой стены, то в поле содержится идентфикатор скопированной записи на стене ее владельца'
    )
    #     copy_post_date = models.DateTimeField(u'Время сообщения-оригинала', null=True)
    #     copy_post_type = models.CharField(max_length=20)
    copy_text = models.TextField(
        u'Комментарий при репосте',
        help_text=
        u'Если запись является копией записи с чужой стены и при её копировании был добавлен комментарий, его текст содержится в данном поле'
    )

    # not in API
    post_source = models.TextField()
    online = models.PositiveSmallIntegerField(null=True)
    reply_count = models.PositiveIntegerField(null=True)

    objects = VkontakteCRUDManager()
    remote = PostRemoteManager(remote_pk=('remote_id', ),
                               methods={
                                   'get': 'get',
                                   'getById': 'getById',
                                   'create': 'post',
                                   'update': 'edit',
                                   'delete': 'delete',
                                   'restore': 'restore',
                               })

    @property
    def reposters(self):
        return [repost.author for repost in self.wall_reposts.all()]

    def __unicode__(self):
        return '%s: %s' % (unicode(self.wall_owner), self.text)

    def save(self, *args, **kwargs):
        # check strings for good encoding
        # there is problems to save users with bad encoded activity strings like user ID=88798245

        #        try:
        #            self.text.encode('utf-16').decode('utf-16')
        #        except UnicodeDecodeError:
        #            self.text = ''

        # поле назначено через API
        if self.copy_owner_id and not self.copy_owner_content_type:
            ct_model = User if self.copy_owner_id > 0 else Group
            self.copy_owner_content_type = ContentType.objects.get_for_model(
                ct_model)
            self.copy_owner = ct_model.remote.fetch(
                ids=[abs(self.copy_owner_id)])[0]

        # save generic fields before saving post
        if self.copy_owner:
            self.copy_owner.save()

        return super(Post, self).save(*args, **kwargs)

    def prepare_create_params(self, **kwargs):
        kwargs.update({
            'owner_id': self.remote_owner_id,
            'friends_only': kwargs.get('friends_only', 0),
            'from_group': kwargs.get('from_group', ''),
            'message': self.text,
            'attachments': self.attachments,
            'services': kwargs.get('services', ''),
            'signed': 1 if self.signer_id else 0,
            'publish_date': kwargs.get('publish_date', ''),
            'lat': kwargs.get('lat', ''),
            'long': kwargs.get('long', ''),
            'place_id': kwargs.get('place_id', ''),
            'post_id': kwargs.get('post_id', '')
        })
        return kwargs

    def prepare_update_params(self, **kwargs):
        return self.prepare_create_params(post_id=self.remote_id_short,
                                          **kwargs)

    def prepare_delete_params(self):
        return {
            'owner_id': self.remote_owner_id,
            'post_id': self.remote_id_short
        }

    def parse_remote_id_from_response(self, response):
        if response:
            return '%s_%s' % (self.remote_owner_id, response['post_id'])
        return None

    def parse(self, response):
        self.raw_json = dict(response)

        for field_name in ['comments', 'likes', 'reposts']:
            if field_name in response and 'count' in response[field_name]:
                setattr(self, field_name, response.pop(field_name)['count'])

        # TODO: may we should move this to save and keep parse queryless
        self.wall_owner = self.get_or_create_group_or_user(
            response.pop('to_id'))[0]
        self.author = self.get_or_create_group_or_user(
            response.pop('from_id'))[0]

        response.pop('attachment', {})
        for attachment in response.pop('attachments', []):
            pass
#            if attachment['type'] == 'poll':
# это можно делать только после сохранения поста, так что тольо через сигналы
#               self.fetch_poll(attachment['poll']['poll_id'])

# TODO: this block broke tests with error
# IntegrityError: new row for relation "vkontakte_wall_post" violates check constraint "vkontakte_wall_post_copy_owner_id_check"
#         if response.get('copy_owner_id'):
#             try:
#                 self.copy_owner_content_type = ContentType.objects.get_for_model(User if response.get('copy_owner_id') > 0 else Group)
#                 self.copy_owner = self.copy_owner_content_type.get_object_for_this_type(remote_id=abs(response.get('copy_owner_id')))
#                 if response.get('copy_post_id'):
#                     self.copy_post = Post.objects.get(remote_id='%s_%s' % (response.get('copy_owner_id'), response.get('copy_post_id')))
#             except ObjectDoesNotExist:
#                 pass

        super(Post, self).parse(response)

        self.remote_id = '%s%s_%s' % (
            ('-' if self.on_group_wall else ''), self.wall_owner.remote_id,
            self.remote_id)

    def fetch_comments(self, *args, **kwargs):
        return Comment.remote.fetch_post(post=self, *args, **kwargs)

    @transaction.commit_on_success
    def fetch_likes(self, source='api', *args, **kwargs):
        if source == 'api':
            return super(Post, self).fetch_likes(*args, **kwargs)
        else:
            return self.fetch_likes_parser(*args, **kwargs)

    @transaction.commit_on_success
    def fetch_likes_parser(self, offset=0):
        '''
        Update and save fields:
            * likes - count of likes
        Update relations:
            * like_users - users, who likes this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'likes/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 120
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 60
            post_data['offset'] = offset

        log.debug('Fetching likes of post "%s" of owner "%s", offset %d' %
                  (self.remote_id, self.wall_owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)

        if offset == 0:
            try:
                self.likes = int(
                    parser.content_bs.find('a', {
                        'id': 'wk_likes_tablikes'
                    }).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning(
                    'Strange markup of first page likes response: "%s"' %
                    parser.content)
            self.like_users.clear()

        #<div class="wk_likes_liker_row inl_bl" id="wk_likes_liker_row722246">
        #  <div class="wk_likes_likerph_wrap" onmouseover="WkView.likesBigphOver(this, 722246)">
        #    <a class="wk_likes_liker_ph" href="/kicolenka">
        #      <img class="wk_likes_liker_img" src="http://cs418825.vk.me/v418825246/6cf8/IBbSfmDz6R8.jpg" width="100" height="100" />
        #    </a>
        #  </div>
        #  <div class="wk_likes_liker_name"><a class="wk_likes_liker_lnk" href="/kicolenka">Оля Киселева</a></div>
        #</div>

        items = parser.add_users(
            users=('div', {
                'class': re.compile(r'^wk_likes_liker_row')
            }),
            user_link=('a', {
                'class': 'wk_likes_liker_lnk'
            }),
            user_photo=('img', {
                'class': 'wk_likes_liker_img'
            }),
            user_add=lambda user: self.like_users.add(user))

        if len(items) == number_on_page:
            self.fetch_likes_parser(offset=offset + number_on_page)
        else:
            return self.like_users.all()

    def fetch_reposts(self, source='api', *args, **kwargs):
        if source == 'api':
            return self.fetch_reposts_api(*args, **kwargs)
        else:
            return self.fetch_reposts_parser(*args, **kwargs)

    def fetch_reposts_api(self, *args, **kwargs):
        self.fetch_instance_reposts(*args, **kwargs)

        # update self.reposts
        # commented, because it's less important count
        #         reposts_count = self.repost_users.get_query_set(only_pk=True).count()
        #         if reposts_count < self.reposts:
        #             log.warning('Fetched ammount of repost users less, than attribute `reposts` of post "%s": %d < %d' % (self.remote_id, reposts_count, self.reposts))
        #         self.reposts = reposts_count
        #         self.save()

        return self.repost_users.all()

    @transaction.commit_on_success
    def fetch_instance_reposts(self, *args, **kwargs):

        resources = self.fetch_repost_items(*args, **kwargs)
        if not resources:
            return Post.objects.none()

        # TODO: still complicated to store reposts objects, may be it's task for another application


#         posts = Post.remote.parse_response(resources)#, extra_fields={'copy_post_id': self.pk})
#         return Post.objects.filter(pk__in=set([Post.remote.get_or_create_from_instance(instance).pk for instance in posts]))

# positive ids -> only users
# TODO: think about how to store reposts by groups
        timestamps = dict([(post['from_id'], post['date'])
                           for post in resources if post['from_id'] > 0])
        ids_new = timestamps.keys()
        ids_current = self.repost_users.get_query_set(
            only_pk=True).using(MASTER_DATABASE).exclude(time_from=None)
        ids_add = set(ids_new).difference(set(ids_current))
        ids_remove = set(ids_current).difference(set(ids_new))

        m2m_model = self.repost_users.through

        # fetch new users
        User.remote.fetch(ids=ids_add, only_expired=True)

        # remove old reposts without time_from
        self.repost_users.get_query_set_through().filter(
            time_from=None).delete()

        # add new reposts
        get_repost_date = lambda id: datetime.fromtimestamp(timestamps[
            id]) if id in timestamps else self.date
        m2m_model.objects.bulk_create([
            m2m_model(
                **{
                    'user_id': id,
                    'post_id': self.pk,
                    'time_from': get_repost_date(id)
                }) for id in ids_add
        ])

        # remove reposts.
        # Commented becouse of .using(MASTER_DATABASE).exclude(time_from=None) filtering for ids_current
        #        m2m_model.objects.filter(post_id=self.pk, user_id__in=ids_remove).update(time_to=datetime.now())
        return

    # не рекомендуется указывать default_count из-за бага паджинации репостов: https://vk.com/wall-51742963_6860
    @fetch_all
    def fetch_repost_items(self, offset=0, count=1000, *args, **kwargs):
        if count > 1000:
            raise ValueError("Parameter 'count' can not be more than 1000")

        # owner_id
        # идентификатор пользователя или сообщества, на стене которого находится запись. Если параметр не задан, то он считается равным идентификатору текущего пользователя.
        # Обратите внимание, идентификатор сообщества в параметре owner_id необходимо указывать со знаком "-" — например, owner_id=-1 соответствует идентификатору сообщества ВКонтакте API (club1)
        kwargs['owner_id'] = self.wall_owner.remote_id
        if isinstance(self.wall_owner, Group):
            kwargs['owner_id'] *= -1
        # post_id
        # идентификатор записи на стене.
        kwargs['post_id'] = self.remote_id.split('_')[1]
        # offset
        # смещение, необходимое для выборки определенного подмножества записей.
        kwargs['offset'] = int(offset)
        # count
        # количество записей, которое необходимо получить.
        # положительное число, по умолчанию 20, максимальное значение 100
        kwargs['count'] = int(count)

        log.debug('Fetching repost users ids of post %s, offset %d' %
                  (self.remote_id, offset))

        response = api_call('wall.getReposts', **kwargs)
        return response['items']

    @transaction.commit_on_success
    def fetch_reposts_parser(self, offset=0):
        '''
        OLD method via parser, may works incorrect
        Update and save fields:
            * reposts - count of reposts
        Update relations
            * repost_users - users, who repost this post
        '''
        post_data = {
            'act': 'show',
            'al': 1,
            'w': 'shares/wall%s' % self.remote_id,
        }

        if offset == 0:
            number_on_page = 40
            post_data['loc'] = 'wall%s' % self.remote_id,
        else:
            number_on_page = 20
            post_data['offset'] = offset

        log.debug('Fetching reposts of post "%s" of owner "%s", offset %d' %
                  (self.remote_id, self.wall_owner, offset))

        parser = VkontakteWallParser().request('/wkview.php', data=post_data)
        if offset == 0:
            try:
                self.reposts = int(
                    parser.content_bs.find('a', {
                        'id': 'wk_likes_tabshares'
                    }).find('nobr').text.split()[0])
                self.save()
            except ValueError:
                return
            except:
                log.warning(
                    'Strange markup of first page shares response: "%s"' %
                    parser.content)
            self.repost_users.clear()

        #<div id="post65120659_2341" class="post post_copy" onmouseover="wall.postOver('65120659_2341')" onmouseout="wall.postOut('65120659_2341')" data-copy="-16297716_126261" onclick="wall.postClick('65120659_2341', event)">
        #  <div class="post_table">
        #    <div class="post_image">
        #      <a class="post_image" href="/vano0ooooo"><img src="/images/camera_c.gif" width="50" height="50"/></a>
        #    </div>
        #      <div class="wall_text"><a class="author" href="/vano0ooooo" data-from-id="65120659">Иван Панов</a> <div id="wpt65120659_2341"></div><table cellpadding="0" cellspacing="0" class="published_by_wrap">

        items = parser.add_users(
            users=('div', {
                'id': re.compile('^post\d'),
                'class': re.compile('^post ')
            }),
            user_link=('a', {
                'class': 'author'
            }),
            user_photo=lambda item: item.find('a', {
                'class': 'post_image'
            }).find('img'),
            user_add=lambda user: self.repost_users.add(user))

        if len(items) == number_on_page:
            self.fetch_reposts(offset=offset + number_on_page)
        else:
            return self.repost_users.all()

    def fetch_statistic(self, *args, **kwargs):
        if 'vkontakte_wall_statistic' not in settings.INSTALLED_APPS:
            raise ImproperlyConfigured(
                "Application 'vkontakte_wall_statistic' not in INSTALLED_APPS")

        from vkontakte_wall_statistic.models import PostStatistic
        return PostStatistic.remote.fetch(post=self, *args, **kwargs)
Exemple #17
0
class Photo(PhotoBase):
    class Meta:
        verbose_name = u'Фотография Одноклассники'
        verbose_name_plural = u'Фотографии Одноклассники'

    remote_pk_field = 'id'

    album = models.ForeignKey(Album, related_name='photos')

    comments_count = models.PositiveIntegerField(default=0)

    created = models.DateTimeField(null=True)

    like_users = ManyToManyHistoryField(User, related_name='like_photos')

    owner_content_type = models.ForeignKey(
        ContentType, related_name='odnoklassniki_photos_owners')
    owner_id = models.BigIntegerField(db_index=True)
    owner = generic.GenericForeignKey('owner_content_type', 'owner_id')

    pic1024max = models.URLField(null=True)
    pic1024x768 = models.URLField(null=True)
    pic128max = models.URLField(null=True)
    pic128x128 = models.URLField(null=True)
    pic180min = models.URLField(null=True)
    pic190x190 = models.URLField(null=True)
    pic240min = models.URLField(null=True)
    pic320min = models.URLField(null=True)
    pic50x50 = models.URLField(null=True)
    pic640x480 = models.URLField(null=True)

    standard_height = models.PositiveIntegerField(default=0)
    standard_width = models.PositiveIntegerField(default=0)

    text = models.TextField()

    remote = PhotoRemoteManager(
        methods={
            'get': 'getPhotos',
            'get_specific': 'getInfo',
            'get_likes': 'getPhotoLikes',
        })

    def fetch_likes(self, **kwargs):
        kwargs['photo_id'] = self.pk

        return super(Photo, self).fetch_likes(**kwargs)

    @property
    def slug(self):
        # Apparently there is no slug for a photo
        return '%s' % (self.album.slug, )

    def __unicode__(self):
        return self.text

    def parse(self, response):
        created = response.pop('created_ms', None)
        if created:
            response[u'created'] = created / 1000

        if response.get('album_id'):
            self.album = Album.objects.get(id=int(response.get('album_id')))

        return super(Photo, self).parse(response)