Exemple #1
0
    def import_video_restrictions(self, automatic_flush=True):
        from rockpack.mainsite.services.video.models import VideoRestriction, VideoInstance, Video, Channel

        with app.test_request_context():
            query = VideoRestriction.query.join(
                VideoInstance,
                VideoInstance.video == VideoRestriction.video
            ).join(Channel, VideoInstance.channel == Channel.id).join(Video, Video.id == VideoRestriction.video).filter(Video.visible == True, Channel.public == True).order_by(VideoInstance.id)

            for relationship in ('allow', 'deny',):
                for instance_id, group in groupby(query.filter(VideoRestriction.relationship == relationship).yield_per(6000).values(VideoInstance.id, VideoRestriction.country), lambda x: x[0]):
                    countries = [c.encode('utf8') for i, c in group]

                    try:
                        self.conn.partial_update(
                            self.indexing['video']['index'],
                            self.indexing['video']['type'],
                            instance_id,
                            "ctx._source.country_restriction.%s = %s" % (relationship, str(countries),)
                        )
                    except pyes.exceptions.ElasticSearchException:
                        pass

                sys.stdout.flush()

                if automatic_flush:
                    self.conn.flush_bulk(forced=True)
Exemple #2
0
    def import_channel_video_counts(self, channel_ids):
        if channel_ids:

            from rockpack.mainsite.services.video.models import Channel, VideoInstance

            with app.test_request_context():
                query = Channel.query.join(
                    VideoInstance,
                    (VideoInstance.channel == Channel.id) &
                    (VideoInstance.deleted == False)
                ).filter(
                    Channel.id.in_(channel_ids)
                ).group_by(
                    Channel.id
                ).values(Channel.id, func.count(Channel.id))

                for channel_id, count in query:
                    try:
                        self.conn.partial_update(
                            self.indexing['channel']['index'],
                            self.indexing['channel']['type'],
                            channel_id,
                            "ctx._source.video_counts = %s" % count
                        )
                    except pyes.exceptions.ElasticSearchException:
                        pass
Exemple #3
0
    def import_users(self, start=None, stop=None, automatic_flush=True):
        from rockpack.mainsite.services.user import models

        with app.test_request_context():
            users = models.User.query.filter(models.User.is_active == True)

            if start:
                users = users.filter(models.User.date_updated.between(start, stop))

            total = users.count()

            app.logger.info('importing %d users', total)

            start_time = time.time()
            count = 1
            for users in users.yield_per(6000):
                api.add_user_to_index(users, bulk=True, no_check=True)
                if app.logger.isEnabledFor(logging.DEBUG):
                    self.print_percent_complete(count, total)
                count += 1

            if automatic_flush:
                self.conn.flush_bulk(forced=True)

            app.logger.debug('finished in %d seconds', time.time() - start_time)
Exemple #4
0
def full_video_import(start=None, stop=None, prefix=None):
    from rockpack.mainsite.services.video import models

    with app.test_request_context():
        videos_to_delete = models.VideoInstance.query.join(
            models.Video,
            models.VideoInstance.video == models.Video.id
        ).filter(
            (models.VideoInstance.deleted == True) |
            (models.Video.visible == False))

        if start:
            videos_to_delete = videos_to_delete.filter(
                (models.VideoInstance.date_updated.between(start, stop)) |
                (models.Video.date_updated.between(start, stop)))

        if prefix:
            videos_to_delete = videos_to_delete.filter(
                models.VideoInstance.id.like(prefix.replace('_', '\\_') + '%'))

        delete = [v[0] for v in videos_to_delete.values(models.VideoInstance.id)]

        if delete:
            api.ESVideo.delete(delete)
            api.ESVideo.flush()

    imp = DBImport()
    imp.import_videos(prefix=prefix, start=start, stop=stop, automatic_flush=False)
    imp.import_dolly_video_owners(prefix=prefix, start=start, stop=stop, automatic_flush=False)
    imp.import_video_stars(prefix=prefix, automatic_flush=False, start=start, stop=stop)
    # Comments support disabled
    # imp.import_comment_counts(prefix=prefix, automatic_flush=False, start=start, stop=stop)

    api.ESVideo.flush()
Exemple #5
0
def full_channel_import(start=None, stop=None):
    from rockpack.mainsite.services.video import models

    with app.test_request_context():
        channels_to_delete = models.Channel.query.filter(
            (models.Channel.public == False) |
            (models.Channel.deleted == True) |
            (models.Channel.visible == False)
        )

        if start:
            channels_to_delete = channels_to_delete.filter(models.Channel.date_updated.between(start, stop))

        delete = [c[0] for c in channels_to_delete.values('id')]

        if delete:
            api.ESChannel.delete(delete)
            api.ESChannel.flush()

    imp = DBImport()
    imp.import_channels(start=start, stop=stop, automatic_flush=False)
    imp.import_average_category(start=start, stop=stop, automatic_flush=False)
    imp.import_video_channel_terms(start=start, stop=stop, automatic_flush=False)

    api.ESChannel.flush()
Exemple #6
0
    def import_dolly_video_owners(self, prefix=None, start=None, stop=None, automatic_flush=True):
        """ Import all the owner attributes of
            a video instance belonging to a channel """

        from rockpack.mainsite.services.video.models import Channel, VideoInstance
        from rockpack.mainsite.services.user.models import User

        with app.test_request_context():
            instances = VideoInstance.query.join(
                Channel,
                (VideoInstance.channel == Channel.id) &
                (Channel.public == True) &
                (Channel.visible == True) &
                (Channel.deleted == False)
            ).join(
                User,
                User.id == Channel.owner
            ).filter(
                VideoInstance.deleted == False
            )

            if prefix:
                instances = instances.filter(VideoInstance.id.like(prefix.replace('_', '\\_') + '%'))

            if start:
                instances = instances.filter(VideoInstance.date_updated.between(start, stop))

            instances = instances.with_entities(VideoInstance, User).order_by(VideoInstance.channel)

            total = instances.count()
            done = 1

            current_user_dict = None
            current_channel = None

            ec = ESVideo.updater(bulk=True)
            for instance, user in instances.yield_per(6000):
                if instance.channel != current_channel:
                    # Duck punch the user on for perf
                    instance.owner_rel = user
                    mapped = ESVideoAttributeMap(instance)
                    current_user_dict = mapped.owner
                    current_channel = instance.channel

                ec.set_document_id(instance.id)
                ec.add_field('owner', current_user_dict)
                ec.update()
                ec.reset()

                if app.logger.isEnabledFor(logging.DEBUG):
                    self.print_percent_complete(done, total)
                done += 1

            app.logger.info('Video owner size: %d', done)

            if automatic_flush:
                self.conn.flush_bulk(forced=True)
Exemple #7
0
    def import_channels(self, start=None, stop=None, automatic_flush=True):
        from rockpack.mainsite.services.video.models import Channel, VideoInstance, Video

        with app.test_request_context():
            channels = Channel.query.filter(
                Channel.public == True,
                Channel.visible == True,
                Channel.deleted == False
            ).options(
                joinedload(Channel.category_rel),
                joinedload(Channel.metas),
                joinedload(Channel.owner_rel)
            )

            if start:
                channels = channels.filter(Channel.date_updated.between(start, stop))

            total = channels.count()

            app.logger.info('importing %d channels', total)

            start_time = time.time()
            ec = ESChannel.inserter(bulk=True)
            count = 1

            query = VideoInstance.query.join(
                Video,
                (Video.id == VideoInstance.video) &
                (Video.visible == True)
            ).filter(VideoInstance.deleted == False).group_by(VideoInstance.channel)

            if start:
                # Restrict the counts selected to the channels we want
                query = query.join(
                    Channel,
                    (Channel.id == VideoInstance.channel) &
                    (Channel.date_updated.between(start, stop))
                )

            query = query.values(VideoInstance.channel, func.count(VideoInstance.id))

            video_counts = dict(query)

            for channel in channels.yield_per(6000):
                channel._video_count = video_counts.get(channel.id) or 0
                ec.insert(channel.id, channel)
                if app.logger.isEnabledFor(logging.DEBUG):
                    self.print_percent_complete(count, total)
                count += 1

            if automatic_flush:
                ec.flush_bulk()

            app.logger.debug('finished in %d seconds', time.time() - start_time)
Exemple #8
0
    def import_search_suggestions(self):
        from rockpack.mainsite.services.video.models import Video

        with app.test_request_context():
            inserter = ESSearchSuggestion.inserter(bulk=True)
            videos = Video.query.filter_by(visible=True).with_entities(
                Video.id,
                literal('video').label('type'),
                Video.title.label('query'),
                Video.star_count.label('weight'),
            )
            for video in videos.yield_per(6000):
                inserter.insert(video.id, video)

            inserter.flush_bulk()
Exemple #9
0
    def import_video_stars(self, prefix=None, automatic_flush=True, start=None, stop=None):
        from rockpack.mainsite.services.user.models import UserActivity

        updated_activity = aliased(UserActivity, name='updated_activity')

        with app.test_request_context():
            query = UserActivity.query.filter(
                UserActivity.action == 'star',
                UserActivity.object_type == 'video_instance'
            ).order_by(
                UserActivity.object_id, UserActivity.date_actioned.desc()
            )

            if start:
                query = query.join(
                    updated_activity,
                    (updated_activity.object_id == UserActivity.object_id) &
                    (updated_activity.date_actioned.between(start, stop)))

            if prefix:
                query = query.filter(UserActivity.object_id.like(prefix.replace('_', '\\_') + '%'))

            ec = ESVideo.updater(bulk=True)
            done = 1
            for instance_id, group in groupby(query.yield_per(200).values(UserActivity.object_id, UserActivity.user), lambda x: x[0]):
                try:
                    ec.set_document_id(instance_id)
                    ec.add_field(
                        'recent_user_stars',
                        list(set([u.encode('utf8') for v, u in group]))[:5]
                    )
                    ec.update()
                except pyes.exceptions.ElasticSearchException:
                    pass
                finally:
                    ec.reset()
                done += 1

            app.logger.info('Video star size: %d', done)

            if automatic_flush:
                self.conn.flush_bulk(forced=True)
Exemple #10
0
def full_user_import(start=None, stop=None):
    from rockpack.mainsite.services.user import models

    with app.test_request_context():
        users_to_delete = models.User.query.filter(models.User.is_active == False)
        if start:
            users_to_delete = users_to_delete.filter(models.User.date_updated.between(start, stop))

        delete = [u[0] for u in users_to_delete.values('id')]

        if delete:
            api.ESUser.delete(delete)
            api.ESUser.flush()

    imp = DBImport()
    imp.import_users(start=start, stop=stop, automatic_flush=False)
    update_user_categories(automatic_flush=False, start=start, stop=stop)
    update_user_subscription_count(start=start, stop=stop, automatic_flush=False)

    api.ESUser.flush()
Exemple #11
0
def install(*args):

    dbfixture = SQLAlchemyFixture(
        env={
            'LocaleData': video_models.Locale,
            'CategoryData': video_models.Category,
            'CategoryTranslationData': video_models.CategoryTranslation,
            'RockpackCoverArtData': RockpackCoverArt,
            'SourceData': video_models.Source,
            'UserData': User,
            'ChannelData': video_models.Channel,
            'ChannelLocaleMetaData': video_models.ChannelLocaleMeta,
            'VideoData': video_models.Video,
            'VideoInstanceData': video_models.VideoInstance,
            'MoodData': video_models.Mood,
            'ExternalTokenData': oauth_models.ExternalToken,
        },
        session=db.session,
    )

    data = dbfixture.data(*args)
    with app.test_request_context():
        data.setup()
        db.session.commit()
Exemple #12
0
    def import_dolly_repin_counts(self, prefix=None, start=None, stop=None, automatic_flush=True):
        from rockpack.mainsite.services.video.models import VideoInstance, Video

        with app.test_request_context():
            child = aliased(VideoInstance, name='child')
            query = readonly_session.query(
                VideoInstance.id,
                VideoInstance.video,
                child.source_channel,
                func.count(VideoInstance.id)
            ).outerjoin(
                child,
                (VideoInstance.video == child.video) &
                (VideoInstance.channel == child.source_channel)
            ).join(
                Video,
                (Video.id == VideoInstance.video) &
                (Video.visible == True) &
                (VideoInstance.deleted == False))

            if prefix:
                query = query.filter(VideoInstance.id.like(prefix.replace('_', '\\_') + '%'))

            if start:
                video_ids = VideoInstance.query.filter(
                    VideoInstance.date_updated.between(start, stop)
                ).with_entities(VideoInstance.video).subquery()

                query = query.filter(Video.id.in_(video_ids))

            query = query.group_by(VideoInstance.id, VideoInstance.video, child.source_channel)

            instance_counts = {}
            influential_index = {}

            total = query.count()
            done = 1

            for _id, video, source_channel, count in query.yield_per(6000):
                # Set the count for the video instance
                instance_counts[(_id, video)] = count
                # If the count is higher for the same video that
                # the previous instance, mark this instance as the
                # influential one for this video
                i_id, i_count = influential_index.get(video, [None, 0])

                # Count will always be at least 1
                # but should really be zero if no children
                if not source_channel and count == 1:
                    count = 0
                if (count > i_count) or\
                        (count == i_count) and not source_channel:
                    influential_index.update({video: (_id, count,)})

                if app.logger.isEnabledFor(logging.DEBUG):
                    self.print_percent_complete(done, total)
                done += 1

            total = len(instance_counts)
            done = 1

            ec = ESVideo.updater(bulk=True)
            for (_id, video), count in instance_counts.iteritems():
                ec.set_document_id(_id)
                ec.add_field('child_instance_count', count)
                ec.add_field('most_influential', True if influential_index.get(video, '')[0] == _id else False)
                ec.update()
                ec.reset()

                if app.logger.isEnabledFor(logging.DEBUG):
                    self.print_percent_complete(done, total)
                done += 1

            if automatic_flush:
                ESVideo.flush()
Exemple #13
0
    def import_videos(self, prefix=None, start=None, stop=None, recent_user_stars=False, automatic_flush=True):
        from rockpack.mainsite.services.video.models import (Channel, Video, VideoInstanceLocaleMeta,
                                                             VideoInstance)
        from rockpack.mainsite.services.user.models import User

        with app.test_request_context():
            query = VideoInstance.query.join(
                Channel,
                Channel.id == VideoInstance.channel
            ).join(Video).outerjoin(
                VideoInstanceLocaleMeta,
                VideoInstance.id == VideoInstanceLocaleMeta.video_instance
            ).outerjoin(
                User,
                (User.id == VideoInstance.original_channel_owner) &
                (User.is_active == True)
            ).options(
                joinedload(VideoInstance.metas)
            ).options(
                joinedload(VideoInstance.video_rel)
            ).options(
                joinedload(VideoInstance.video_channel)
            ).options(
                joinedload(VideoInstance.original_channel_owner_rel)
            ).filter(
                VideoInstance.deleted == False,
                Video.visible == True
            )

            if prefix:
                query = query.filter(VideoInstance.id.like(prefix.replace('_', '\\_') + '%'))

            if start:
                query = query.filter(VideoInstance.date_updated.between(start, stop))

            total = query.count()
            app.logger.info('importing %d videos', total)
            start_time = time.time()
            done = 1

            ev = ESVideo.inserter(bulk=True)
            channel_ids = []
            for v in query.yield_per(6000):
                mapped = ESVideoAttributeMap(v)
                rep = dict(
                    id=mapped.id,
                    public=mapped.public,
                    video=mapped.video,
                    title=mapped.title,
                    channel=mapped.channel,
                    channel_title=mapped.channel_title,
                    category=mapped.category,
                    date_added=mapped.date_added,
                    position=mapped.position,
                    locales=mapped.locales,
                    recent_user_stars=mapped.recent_user_stars(empty=not recent_user_stars),
                    country_restriction=mapped.country_restriction(empty=True),
                    comments=mapped.comments(empty=True),
                    child_instance_count=mapped.child_instance_count,
                    link_url=mapped.link_url,
                    link_title=mapped.link_title,
                    tags=mapped.tags,
                    is_favourite=mapped.is_favourite,
                    most_influential=mapped.most_influential,
                    original_channel_owner=mapped.original_channel_owner,
                    label=mapped.label,
                )
                ev.manager.indexer.insert(v.id, rep)

                if app.logger.isEnabledFor(logging.DEBUG):
                    self.print_percent_complete(done, total)
                done += 1

                if start and v.channel not in channel_ids:
                    channel_ids.append(v.channel)

            if start:
                # We'll want to update channel counts that
                # may not get trigger elsewhere
                self.import_channel_video_counts(channel_ids)

            if automatic_flush:
                ev.flush_bulk()

            app.logger.debug('finished in %d seconds', time.time() - start_time)
Exemple #14
0
    def test_video_comments(self):
        with app.test_request_context():
            user_id = self.create_test_user().id
            with self.app.test_client() as client:
                # create new channel
                r = client.post(
                    '/ws/{}/channels/'.format(user_id),
                    data=json.dumps(dict(
                        title='test',
                        description='test',
                        category='',
                        cover='',
                        public=True,
                    )),
                    content_type='application/json',
                    headers=[get_auth_header(user_id)]
                )
                self.assertEquals(r.status_code, 201)
                channel_id = json.loads(r.data)['id']

                # add videos
                r = client.put(
                    '/ws/{}/channels/{}/videos/'.format(user_id, channel_id),
                    data=json.dumps([
                        VideoInstanceData.video_instance1.id,
                        VideoInstanceData.video_instance2.id,
                    ]),
                    content_type='application/json',
                    headers=[get_auth_header(user_id)]
                )
                self.assertEquals(r.status_code, 204)

                # add comment
                instance_data = dict(userid=user_id, channelid=channel_id)

                r = client.get(
                    '/ws/{userid}/channels/{channelid}/'.format(**instance_data),
                    content_type='application/json',
                    headers=[get_auth_header(user_id)])

                instance_data['videoid'] = json.loads(r.data)['videos']['items'][0]['id']

                r = client.post(
                    '/ws/{userid}/channels/{channelid}/videos/{videoid}/comments/'.format(**instance_data),
                    data=json.dumps(dict(comment="this is a comment")),
                    content_type='application/json',
                    headers=[get_auth_header(user_id)]
                )
                self.assertEquals(r.status_code, 201)

                self.wait_for_es()

                v = VideoSearch('en-gb')
                v.add_id(instance_data['videoid'])
                instance = v.videos()[0]
                self.assertEquals(instance['comments']['total'], 1)

                # delete comment
                r = client.get(
                    '/ws/{userid}/channels/{channelid}/videos/{videoid}/comments/'.format(**instance_data),
                    content_type='application/json',
                    headers=[get_auth_header(user_id)]
                )

                comment_id = json.loads(r.data)['comments']['items'][0]['id']
                instance_data.update({'commentid': comment_id})

                r = client.delete(
                    '/ws/{userid}/channels/{channelid}/videos/{videoid}/comments/{commentid}/'.format(**instance_data),
                    content_type='application/json',
                    headers=[get_auth_header(user_id)]
                )

                self.wait_for_es()

                v = VideoSearch('en-gb')
                v.add_id(instance_data['videoid'])
                instance = v.videos()[0]
                self.assertEquals(instance['comments']['total'], 0)