Пример #1
0
    def forwards(self, orm):
        if db.dry_run:
            return        
        
        try:
            from utils.redis_utils import default_connection
            from statistic import get_fetch_subtitles_key
        except ImportError:
            if settings.DEBUG:
                return
            raise Exception('Some redis utilits is unavailable, maybe this migration was not updated after refactoring. You can ignore this migration with: python manage.py migrate statistic 0002 --fake, but all statistic data will be lost.')

        try:
            default_connection.ping()
        except ConnectionError:
            if settings.DEBUG:
                return
            raise Exception('Redis server is unavailable. You can ignore this migration with: python manage.py migrate statistic 0002 --fake, but all statistic data will be lost.')
        
        print 'Total count of rows: ', orm.SubtitleFetchStatistic.objects.count()
        
        keys = []
        for item in orm.SubtitleFetchStatistic.objects.order_by('id'):
            
            if not item.pk % 1000:
                print item.pk
                
            key = get_fetch_subtitles_key(item.video, item.language, item.created)
            if not key in keys:
                default_connection.delete(key)
                keys.append(key)
            default_connection.incr(key)
Пример #2
0
    def handle(self, *args, **kwargs):
        try:
            sub_fetch_keys_set.r.ping()
        except:
            if settings.DEBUG:
                raise
            self.handle_error('Statistic update error', '', sys.exc_info())     
            return
        
        count = sub_fetch_keys_set.scard()
        
        while count:
            count -= 1
            key = sub_fetch_keys_set.spop()
            
            if not key:
                break
            
            print 'Handle key: %s' % key
            
            parts = key.split(':')
            d = date(int(parts[-1]), int(parts[-2]), int(parts[-3]))
            
            if len(parts) == 6:
                lang = parts[2]
            else:
                lang = ''
            
            try:
                video = Video.objects.get(video_id=parts[1])
            except Video.DoesNotExist:
                print 'Video does not exist'
                default_connection.delete(key)
                continue
            
            counter_obj, created = SubtitleFetchCounters.objects.get_or_create(date=d, video=video, language=lang)
            counter_obj.count += int(default_connection.getset(key, 0))
            counter_obj.save()
            
        count = changed_video_set.scard()
        
        while count:
            count -= 1

            video_id = changed_video_set.spop()

            if not video_id:
                break
            
            print 'Update statistic for video: %s' % video_id
            
            subtitles_fetched_counter = Video.subtitles_fetched_counter(video_id, True)
            widget_views_counter = Video.widget_views_counter(video_id, True)
            view_counter = Video.view_counter(video_id, True)
            
            Video.objects.filter(video_id=video_id).update(view_count=F('view_count')+view_counter.getset(0))
            Video.objects.update(widget_views_count=F('widget_views_count')+widget_views_counter.getset(0))
            Video.objects.update(subtitles_fetched_count=F('subtitles_fetched_count')+subtitles_fetched_counter.getset(0))
Пример #3
0
    def _test_redis(self):
        print '=== REDIS ==='
        assert default_connection.ping(), u'Redis is unavailable'

        val = str(random.random())
        key = 'test-redis-%s' % base64.b64encode(str(random.random()))
        
        default_connection.set(key, val)
        assert val == default_connection.get(key), u'Redis is unavailable. Can\'t get value'
        
        default_connection.delete(key)
        assert default_connection.get(key) is None, u'Redis is unavailable. Can\'t delete value'
                
        print 'OK'
        print
Пример #4
0
    def _test_redis(self):
        print '=== REDIS ==='
        assert default_connection.ping(), u'Redis is unavailable'

        val = str(random.random())
        key = 'test-redis-%s' % base64.b64encode(str(random.random()))
        
        default_connection.set(key, val)
        assert val == default_connection.get(key), u'Redis is unavailable. Can\'t get value'
        
        default_connection.delete(key)
        assert default_connection.get(key) is None, u'Redis is unavailable. Can\'t delete value'
                
        print 'OK'
        print
Пример #5
0
    def handle(self, *args, **kwargs):
        try:
            sub_fetch_keys_set.r.ping()
        except:
            if settings.DEBUG:
                raise
            self.handle_error('Statistic update error', '', sys.exc_info())
            return

        count = sub_fetch_keys_set.scard()

        while count:
            count -= 1
            key = sub_fetch_keys_set.spop()

            if not key:
                break

            print 'Handle key: %s' % key

            parts = key.split(':')
            d = date(int(parts[-1]), int(parts[-2]), int(parts[-3]))

            if len(parts) == 6:
                lang = parts[2]
            else:
                lang = ''

            try:
                video = Video.objects.get(video_id=parts[1])
            except Video.DoesNotExist:
                print 'Video does not exist'
                default_connection.delete(key)
                continue

            counter_obj, created = SubtitleFetchCounters.objects.get_or_create(
                date=d, video=video, language=lang)
            counter_obj.count += int(default_connection.getset(key, 0))
            counter_obj.save()

        count = changed_video_set.scard()

        while count:
            count -= 1

            video_id = changed_video_set.spop()

            if not video_id:
                break

            print 'Update statistic for video: %s' % video_id

            subtitles_fetched_counter = Video.subtitles_fetched_counter(
                video_id, True)
            widget_views_counter = Video.widget_views_counter(video_id, True)
            view_counter = Video.view_counter(video_id, True)

            Video.objects.filter(video_id=video_id).update(
                view_count=F('view_count') + view_counter.getset(0))
            Video.objects.update(widget_views_count=F('widget_views_count') +
                                 widget_views_counter.getset(0))
            Video.objects.update(
                subtitles_fetched_count=F('subtitles_fetched_count') +
                subtitles_fetched_counter.getset(0))