def forwards(self, orm):
if db.dry_run:
return
try:
from utils.redis_utils import default_connection
from statistic import get_fetch_subtitles_key
except ImportError:
if settings.DEBUG:
return
raise Exception('Some redis utilits is unavailable, maybe this migration was not updated after refactoring. You can ignore this migration with: python manage.py migrate statistic 0002 --fake, but all statistic data will be lost.')
try:
default_connection.ping()
except ConnectionError:
if settings.DEBUG:
return
raise Exception('Redis server is unavailable. You can ignore this migration with: python manage.py migrate statistic 0002 --fake, but all statistic data will be lost.')
print 'Total count of rows: ', orm.SubtitleFetchStatistic.objects.count()
keys = []
for item in orm.SubtitleFetchStatistic.objects.order_by('id'):
if not item.pk % 1000:
print item.pk
key = get_fetch_subtitles_key(item.video, item.language, item.created)
if not key in keys:
default_connection.delete(key)
keys.append(key)
default_connection.incr(key)
def handle(self, *args, **kwargs):
try:
sub_fetch_keys_set.r.ping()
except:
if settings.DEBUG:
raise
self.handle_error('Statistic update error', '', sys.exc_info())
return
count = sub_fetch_keys_set.scard()
while count:
count -= 1
key = sub_fetch_keys_set.spop()
if not key:
break
print 'Handle key: %s' % key
parts = key.split(':')
d = date(int(parts[-1]), int(parts[-2]), int(parts[-3]))
if len(parts) == 6:
lang = parts[2]
else:
lang = ''
try:
video = Video.objects.get(video_id=parts[1])
except Video.DoesNotExist:
print 'Video does not exist'
default_connection.delete(key)
continue
counter_obj, created = SubtitleFetchCounters.objects.get_or_create(date=d, video=video, language=lang)
counter_obj.count += int(default_connection.getset(key, 0))
counter_obj.save()
count = changed_video_set.scard()
while count:
count -= 1
video_id = changed_video_set.spop()
if not video_id:
break
print 'Update statistic for video: %s' % video_id
subtitles_fetched_counter = Video.subtitles_fetched_counter(video_id, True)
widget_views_counter = Video.widget_views_counter(video_id, True)
view_counter = Video.view_counter(video_id, True)
Video.objects.filter(video_id=video_id).update(view_count=F('view_count')+view_counter.getset(0))
Video.objects.update(widget_views_count=F('widget_views_count')+widget_views_counter.getset(0))
Video.objects.update(subtitles_fetched_count=F('subtitles_fetched_count')+subtitles_fetched_counter.getset(0))
def _test_redis(self):
print '=== REDIS ==='
assert default_connection.ping(), u'Redis is unavailable'
val = str(random.random())
key = 'test-redis-%s' % base64.b64encode(str(random.random()))
default_connection.set(key, val)
assert val == default_connection.get(key), u'Redis is unavailable. Can\'t get value'
default_connection.delete(key)
assert default_connection.get(key) is None, u'Redis is unavailable. Can\'t delete value'
print 'OK'
print
def _test_redis(self):
print '=== REDIS ==='
assert default_connection.ping(), u'Redis is unavailable'
val = str(random.random())
key = 'test-redis-%s' % base64.b64encode(str(random.random()))
default_connection.set(key, val)
assert val == default_connection.get(key), u'Redis is unavailable. Can\'t get value'
default_connection.delete(key)
assert default_connection.get(key) is None, u'Redis is unavailable. Can\'t delete value'
print 'OK'
print
def handle(self, *args, **kwargs):
try:
sub_fetch_keys_set.r.ping()
except:
if settings.DEBUG:
raise
self.handle_error('Statistic update error', '', sys.exc_info())
return
count = sub_fetch_keys_set.scard()
while count:
count -= 1
key = sub_fetch_keys_set.spop()
if not key:
break
print 'Handle key: %s' % key
parts = key.split(':')
d = date(int(parts[-1]), int(parts[-2]), int(parts[-3]))
if len(parts) == 6:
lang = parts[2]
else:
lang = ''
try:
video = Video.objects.get(video_id=parts[1])
except Video.DoesNotExist:
print 'Video does not exist'
default_connection.delete(key)
continue
counter_obj, created = SubtitleFetchCounters.objects.get_or_create(
date=d, video=video, language=lang)
counter_obj.count += int(default_connection.getset(key, 0))
counter_obj.save()
count = changed_video_set.scard()
while count:
count -= 1
video_id = changed_video_set.spop()
if not video_id:
break
print 'Update statistic for video: %s' % video_id
subtitles_fetched_counter = Video.subtitles_fetched_counter(
video_id, True)
widget_views_counter = Video.widget_views_counter(video_id, True)
view_counter = Video.view_counter(video_id, True)
Video.objects.filter(video_id=video_id).update(
view_count=F('view_count') + view_counter.getset(0))
Video.objects.update(widget_views_count=F('widget_views_count') +
widget_views_counter.getset(0))
Video.objects.update(
subtitles_fetched_count=F('subtitles_fetched_count') +
subtitles_fetched_counter.getset(0))
