def forwards(self, orm): if db.dry_run: return try: from utils.redis_utils import default_connection from statistic import get_fetch_subtitles_key except ImportError: if settings.DEBUG: return raise Exception('Some redis utilits is unavailable, maybe this migration was not updated after refactoring. You can ignore this migration with: python manage.py migrate statistic 0002 --fake, but all statistic data will be lost.') try: default_connection.ping() except ConnectionError: if settings.DEBUG: return raise Exception('Redis server is unavailable. You can ignore this migration with: python manage.py migrate statistic 0002 --fake, but all statistic data will be lost.') print 'Total count of rows: ', orm.SubtitleFetchStatistic.objects.count() keys = [] for item in orm.SubtitleFetchStatistic.objects.order_by('id'): if not item.pk % 1000: print item.pk key = get_fetch_subtitles_key(item.video, item.language, item.created) if not key in keys: default_connection.delete(key) keys.append(key) default_connection.incr(key)
def handle(self, *args, **kwargs): try: sub_fetch_keys_set.r.ping() except: if settings.DEBUG: raise self.handle_error('Statistic update error', '', sys.exc_info()) return count = sub_fetch_keys_set.scard() while count: count -= 1 key = sub_fetch_keys_set.spop() if not key: break print 'Handle key: %s' % key parts = key.split(':') d = date(int(parts[-1]), int(parts[-2]), int(parts[-3])) if len(parts) == 6: lang = parts[2] else: lang = '' try: video = Video.objects.get(video_id=parts[1]) except Video.DoesNotExist: print 'Video does not exist' default_connection.delete(key) continue counter_obj, created = SubtitleFetchCounters.objects.get_or_create(date=d, video=video, language=lang) counter_obj.count += int(default_connection.getset(key, 0)) counter_obj.save() count = changed_video_set.scard() while count: count -= 1 video_id = changed_video_set.spop() if not video_id: break print 'Update statistic for video: %s' % video_id subtitles_fetched_counter = Video.subtitles_fetched_counter(video_id, True) widget_views_counter = Video.widget_views_counter(video_id, True) view_counter = Video.view_counter(video_id, True) Video.objects.filter(video_id=video_id).update(view_count=F('view_count')+view_counter.getset(0)) Video.objects.update(widget_views_count=F('widget_views_count')+widget_views_counter.getset(0)) Video.objects.update(subtitles_fetched_count=F('subtitles_fetched_count')+subtitles_fetched_counter.getset(0))
def _test_redis(self): print '=== REDIS ===' assert default_connection.ping(), u'Redis is unavailable' val = str(random.random()) key = 'test-redis-%s' % base64.b64encode(str(random.random())) default_connection.set(key, val) assert val == default_connection.get(key), u'Redis is unavailable. Can\'t get value' default_connection.delete(key) assert default_connection.get(key) is None, u'Redis is unavailable. Can\'t delete value' print 'OK' print
def handle(self, *args, **kwargs): try: sub_fetch_keys_set.r.ping() except: if settings.DEBUG: raise self.handle_error('Statistic update error', '', sys.exc_info()) return count = sub_fetch_keys_set.scard() while count: count -= 1 key = sub_fetch_keys_set.spop() if not key: break print 'Handle key: %s' % key parts = key.split(':') d = date(int(parts[-1]), int(parts[-2]), int(parts[-3])) if len(parts) == 6: lang = parts[2] else: lang = '' try: video = Video.objects.get(video_id=parts[1]) except Video.DoesNotExist: print 'Video does not exist' default_connection.delete(key) continue counter_obj, created = SubtitleFetchCounters.objects.get_or_create( date=d, video=video, language=lang) counter_obj.count += int(default_connection.getset(key, 0)) counter_obj.save() count = changed_video_set.scard() while count: count -= 1 video_id = changed_video_set.spop() if not video_id: break print 'Update statistic for video: %s' % video_id subtitles_fetched_counter = Video.subtitles_fetched_counter( video_id, True) widget_views_counter = Video.widget_views_counter(video_id, True) view_counter = Video.view_counter(video_id, True) Video.objects.filter(video_id=video_id).update( view_count=F('view_count') + view_counter.getset(0)) Video.objects.update(widget_views_count=F('widget_views_count') + widget_views_counter.getset(0)) Video.objects.update( subtitles_fetched_count=F('subtitles_fetched_count') + subtitles_fetched_counter.getset(0))