def test_delete_int(self):
key = "testkey1"
value = 123
SettingProperties.set_int(key, value)
SettingProperties.delete_key(key)
retreived_value = SettingProperties.get_int(key, 0)
self.assertEqual(0, retreived_value)
def test_delete_string(self):
key = "testkey1"
value = "testval"
SettingProperties.set_string(key, value)
SettingProperties.delete_key(key)
retreived_value = SettingProperties.get_string(key, "default")
self.assertEqual("default", retreived_value)
def handle(self, *args, **options):
# check if cron already running
prop, created = SettingProperties.objects \
.get_or_create(key='oppia_summary_cron_lock',
int_value=1)
if not created:
self.stdout.write("Oppia summary cron is already running")
return
try:
SettingProperties.objects.get(key='oppia_cron_lock')
self.stdout.write("Oppia cron is already running")
SettingProperties.delete_key('oppia_summary_cron_lock')
return
except SettingProperties.DoesNotExist:
# do nothing
pass
if options['fromstart']:
self.update_summaries(0, 0, options['fromstart'])
else:
# get last tracker and points PKs processed
last_tracker_pk = SettingProperties \
.get_property('last_tracker_pk', 0)
last_points_pk = SettingProperties \
.get_property('last_points_pk', 0)
self.update_summaries(last_tracker_pk, last_points_pk)
def test_summary_cron_locked_cron_locked(self):
# set locked
SettingProperties.set_int('oppia_cron_lock', 1)
lock = SettingProperties.get_int('oppia_cron_lock', 0)
self.assertEqual(lock, 1)
SettingProperties.set_int('oppia_summary_cron_lock', 1)
lock = SettingProperties.get_int('oppia_summary_cron_lock', 0)
self.assertEqual(lock, 1)
call_command('update_summaries', stdout=StringIO())
# check new details on pks
# cron is locked so nothing should happen
tracker_id = SettingProperties.get_int('last_tracker_pk', 0)
self.assertEqual(tracker_id, 0)
# unlock
SettingProperties.delete_key('oppia_summary_cron_lock')
SettingProperties.delete_key('oppia_cron_lock')
# check unlocked again
lock = SettingProperties.get_int('oppia_summary_cron_lock', 999)
self.assertEqual(lock, 999)
lock = SettingProperties.get_int('oppia_cron_lock', 999)
self.assertEqual(lock, 999)
def handle(self, *args, **options):
if options['hours']:
hours = options['hours']
else:
hours = 0
# check if cron already running
prop, created = SettingProperties.objects.get_or_create(key='oppia_cron_lock', int_value=1)
if not created:
self.stdout.write("Oppia cron is already running")
return
now = time.time()
path = os.path.join(settings.COURSE_UPLOAD_DIR, "temp")
if os.path.exists(path):
self.stdout.write('Cleaning up: ' + path)
for f in os.listdir(path):
f = os.path.join(path, f)
if os.stat(f).st_mtime < now - 3600 * 6:
self.stdout.write("deleting: {file}".format(file=f))
if os.path.isfile(f):
os.remove(f)
else:
self.stdout.write('{path} does not exist. Don\'t need to clean it'.format(path=path))
from oppia.awards import courses_completed
courses_completed(int(hours))
# create and new media images
call_command('generate_media_images')
SettingProperties.set_string('oppia_cron_last_run', timezone.now())
SettingProperties.delete_key('oppia_cron_lock')
def handle(self, *args, **options):
if options['hours']:
hours=options['hours']
else:
hours=0
#check if cron already running
prop, created = SettingProperties.objects.get_or_create(key='oppia_cron_lock',int_value=1)
if not created:
print("Oppia cron is already running")
return
now = time.time()
path = os.path.join(settings.COURSE_UPLOAD_DIR, "temp")
if os.path.exists(path):
print('Cleaning up: ' + path)
for f in os.listdir(path):
f = os.path.join(path, f)
if os.stat(f).st_mtime < now - 3600 * 6:
print("deleting: {file}".format(file=f))
if os.path.isfile(f):
os.remove(f)
else:
print('{path} does not exist. Don\'t need to clean it'.format(path=path))
from oppia.awards import courses_completed
courses_completed(int(hours))
# create and new media images
call_command('generate_media_images')
SettingProperties.set_string('oppia_cron_last_run', timezone.now())
SettingProperties.delete_key('oppia_cron_lock')
def handle(self, *args, **options):
if options['hours']:
hours = options['hours']
else:
hours = 0
# check if cron already running
prop, created = SettingProperties.objects \
.get_or_create(key='oppia_cron_lock', int_value=1)
if not created:
self.stdout.write("Oppia cron is already running")
return
now = time.time()
path = os.path.join(settings.COURSE_UPLOAD_DIR, "temp")
if os.path.exists(path):
self.stdout.write('Cleaning up: ' + path)
for f in os.listdir(path):
f = os.path.join(path, f)
if os.stat(f).st_mtime < now - 3600 * 6:
self.stdout.write("deleting: {file}".format(file=f))
if os.path.isfile(f):
os.remove(f)
else:
self.stdout \
.write('{path} does not exist. Don\'t need to clean it'
.format(path=path))
from oppia.awards import courses_completed
courses_completed(int(hours))
# generate pdf certificates
call_command('generate_certificates')
# clear any expired sessions
call_command('clearsessions')
# add any missing api keys for Tastypie
call_command('backfill_api_keys')
SettingProperties.set_string('oppia_cron_last_run', timezone.now())
SettingProperties.delete_key('oppia_cron_lock')
# server registration
# deliberately left until after removing cron lock in case issue with
# connecting to implementation server
call_command('update_server_registration')
def test_summary_cron_locked(self):
# set lock not
SettingProperties.set_int('oppia_summary_cron_lock', 1)
lock = SettingProperties.get_int('oppia_summary_cron_lock', 0)
self.assertEqual(lock, 1)
update_summaries()
# check new details on pks
# cron is locked so nothing should happen
tracker_id = SettingProperties.get_int('last_tracker_pk', 0)
self.assertEqual(tracker_id, 0)
#unlock
SettingProperties.delete_key('oppia_summary_cron_lock')
#check unlocked again
lock = SettingProperties.get_int('oppia_summary_cron_lock', 999)
self.assertEqual(lock, 999)
def update_summaries(self,
last_tracker_pk=0,
last_points_pk=0,
fromstart=False):
SettingProperties.set_string('oppia_summary_cron_last_run',
timezone.now())
# get last tracker and points PKs to be processed
# (to avoid leaving some out if new trackers arrive while processing)
try:
newest_tracker_pk = Tracker.objects.latest('id').id
newest_points_pk = Points.objects.latest('id').id
except Tracker.DoesNotExist:
self.stdout.write("Tracker table is empty. Aborting cron...")
SettingProperties.delete_key('oppia_summary_cron_lock')
return
except Points.DoesNotExist:
newest_points_pk = last_points_pk
print('Last tracker processed: %d\nNewest tracker: %d\n' %
(last_tracker_pk, newest_tracker_pk))
if last_tracker_pk >= newest_tracker_pk:
self.stdout.write('No new trackers to process. Aborting cron...')
SettingProperties.delete_key('oppia_summary_cron_lock')
return
self.update_user_course_summary(last_tracker_pk, newest_tracker_pk,
last_points_pk, newest_points_pk)
self.update_course_daily_stats(last_tracker_pk, newest_tracker_pk)
self.update_user_points_summary(last_points_pk, newest_points_pk)
self.update_daily_active_users(last_tracker_pk, newest_tracker_pk)
# update last tracker and points PKs with the last one processed
SettingProperties.objects \
.update_or_create(key='last_tracker_pk',
defaults={"int_value":
newest_tracker_pk})
SettingProperties.objects.update_or_create(
key='last_points_pk', defaults={"int_value": newest_points_pk})
SettingProperties.delete_key('oppia_summary_cron_lock')
def update_summaries(last_tracker_pk=0, last_points_pk=0):
from django.contrib.auth.models import User
from django.db.models import Count
from oppia.models import Tracker, Points, Course
from settings.models import SettingProperties
from summary.models import UserCourseSummary, CourseDailyStats, UserPointsSummary
#check if cron already running
prop, created = SettingProperties.objects.get_or_create(
key='oppia_summary_cron_lock', int_value=1)
if not created:
print("Oppia summary cron is already running")
return
SettingProperties.set_string('oppia_summary_cron_last_run', timezone.now())
# get last tracker and points PKs to be processed
# (to avoid leaving some out if new trackers arrive while processing)
try:
newest_tracker_pk = Tracker.objects.latest('id').id
newest_points_pk = Points.objects.latest('id').id
except Tracker.DoesNotExist:
print("Tracker table is empty. Aborting cron...")
SettingProperties.delete_key('oppia_summary_cron_lock')
return
except Points.DoesNotExist:
newest_points_pk = last_points_pk
print('Last tracker processed: %d\nNewest tracker: %d\n' %
(last_tracker_pk, newest_tracker_pk))
if last_tracker_pk >= newest_tracker_pk:
print('No new trackers to process. Aborting cron...')
SettingProperties.delete_key('oppia_summary_cron_lock')
return
first_tracker = (last_tracker_pk == 0)
first_points = (last_points_pk == 0)
# If we are calculating from the start, delete previous summary calculations
if first_tracker:
UserCourseSummary.objects.all().delete()
CourseDailyStats.objects.all().delete()
if first_points:
UserPointsSummary.objects.all().delete()
# get different (distinct) user/courses involved
user_courses = Tracker.objects \
.filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \
.exclude(course__isnull=True) \
.values('course', 'user').distinct()
total_users = user_courses.count()
print('%d different user/courses to process.' % total_users)
count = 1
for uc_tracker in user_courses:
print('processing user/course trackers... (%d/%d)' %
(count, total_users))
user = User.objects.get(pk=uc_tracker['user'])
course = Course.objects.get(pk=uc_tracker['course'])
user_course, created = UserCourseSummary.objects.get_or_create(
course=course, user=user)
user_course.update_summary(last_tracker_pk=last_tracker_pk,
last_points_pk=last_points_pk,
newest_tracker_pk=newest_tracker_pk,
newest_points_pk=newest_points_pk)
count += 1
# get different (distinct) courses/dates involved
course_daily_type_logs = Tracker.objects \
.filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \
.exclude(course__isnull=True) \
.extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \
.values('course', 'day', 'month', 'year', 'type') \
.annotate(total=Count('type')) \
.order_by('day')
total_logs = course_daily_type_logs.count()
print('%d different courses/dates/types to process.' % total_logs)
count = 0
for type_log in course_daily_type_logs:
day = date(type_log['year'], type_log['month'], type_log['day'])
course = Course.objects.get(pk=type_log['course'])
stats, created = CourseDailyStats.objects.get_or_create(
course=course, day=day, type=type_log['type'])
stats.total = (0 if first_tracker else stats.total) + type_log['total']
stats.save()
count += 1
print(count)
# get different (distinct) search logs involved
search_daily_logs = Tracker.objects \
.filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk, user__is_staff=False, type='search') \
.extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \
.values('day', 'month', 'year') \
.annotate(total=Count('id')) \
.order_by('day')
print('%d different search/dates to process.' % search_daily_logs.count())
for search_log in search_daily_logs:
day = date(search_log['year'], search_log['month'], search_log['day'])
stats, created = CourseDailyStats.objects.get_or_create(course=None,
day=day,
type='search')
stats.total = (0
if first_tracker else stats.total) + search_log['total']
stats.save()
# get different (distinct) user/points involved
users_points = Points.objects \
.filter(pk__gt=last_points_pk, pk__lte=newest_points_pk) \
.values('user').distinct()
total_users = users_points.count()
print('%d different user/points to process.' % total_users)
for user_points in users_points:
user = User.objects.get(pk=user_points['user'])
points, created = UserPointsSummary.objects.get_or_create(user=user)
points.update_points(last_points_pk=last_points_pk,
newest_points_pk=newest_points_pk)
# update last tracker and points PKs with the last one processed
SettingProperties.objects.update_or_create(
key='last_tracker_pk', defaults={"int_value": newest_tracker_pk})
SettingProperties.objects.update_or_create(
key='last_points_pk', defaults={"int_value": newest_points_pk})
SettingProperties.delete_key('oppia_summary_cron_lock')
def update_summaries(last_tracker_pk=0, last_points_pk=0):
from django.contrib.auth.models import User
from django.db.models import Count
from oppia.models import Tracker, Points, Course
from settings.models import SettingProperties
from summary.models import UserCourseSummary, CourseDailyStats, UserPointsSummary
#check if cron already running
prop, created = SettingProperties.objects.get_or_create(key='oppia_summary_cron_lock',int_value=1)
if not created:
print("Oppia summary cron is already running")
return
SettingProperties.set_string('oppia_summary_cron_last_run', timezone.now())
# get last tracker and points PKs to be processed
# (to avoid leaving some out if new trackers arrive while processing)
try:
newest_tracker_pk = Tracker.objects.latest('id').id
newest_points_pk = Points.objects.latest('id').id
except Tracker.DoesNotExist:
print("Tracker table is empty. Aborting cron...")
SettingProperties.delete_key('oppia_summary_cron_lock')
return
except Points.DoesNotExist:
newest_points_pk = last_points_pk
print ('Last tracker processed: %d\nNewest tracker: %d\n' % (last_tracker_pk, newest_tracker_pk))
if last_tracker_pk >= newest_tracker_pk:
print('No new trackers to process. Aborting cron...')
SettingProperties.delete_key('oppia_summary_cron_lock')
return
first_tracker = (last_tracker_pk == 0)
first_points = (last_points_pk == 0)
# If we are calculating from the start, delete previous summary calculations
if first_tracker:
UserCourseSummary.objects.all().delete()
CourseDailyStats.objects.all().delete()
if first_points:
UserPointsSummary.objects.all().delete()
# get different (distinct) user/courses involved
user_courses = Tracker.objects \
.filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \
.exclude(course__isnull=True) \
.values('course', 'user').distinct()
total_users = user_courses.count()
print('%d different user/courses to process.' % total_users)
count = 1
for uc_tracker in user_courses:
print('processing user/course trackers... (%d/%d)' % (count, total_users))
user = User.objects.get(pk=uc_tracker['user'])
course = Course.objects.get(pk=uc_tracker['course'])
user_course, created = UserCourseSummary.objects.get_or_create(course=course, user=user)
user_course.update_summary(
last_tracker_pk=last_tracker_pk, last_points_pk=last_points_pk,
newest_tracker_pk=newest_tracker_pk, newest_points_pk=newest_points_pk)
count += 1
# get different (distinct) courses/dates involved
course_daily_type_logs = Tracker.objects \
.filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \
.exclude(course__isnull=True) \
.extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \
.values('course', 'day', 'month', 'year', 'type') \
.annotate(total=Count('type')) \
.order_by('day')
total_logs = course_daily_type_logs.count()
print('%d different courses/dates/types to process.' % total_logs)
count = 0
for type_log in course_daily_type_logs:
day = date(type_log['year'], type_log['month'], type_log['day'])
course = Course.objects.get(pk=type_log['course'])
stats, created = CourseDailyStats.objects.get_or_create(course=course, day=day, type=type_log['type'])
stats.total = (0 if first_tracker else stats.total) + type_log['total']
stats.save()
count += 1
print(count)
# get different (distinct) search logs involved
search_daily_logs = Tracker.objects \
.filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk, user__is_staff=False, type='search') \
.extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \
.values('day', 'month', 'year') \
.annotate(total=Count('id')) \
.order_by('day')
print('%d different search/dates to process.' % search_daily_logs.count())
for search_log in search_daily_logs:
day = date(search_log['year'], search_log['month'], search_log['day'])
stats, created = CourseDailyStats.objects.get_or_create(course=None, day=day, type='search')
stats.total = (0 if first_tracker else stats.total) + search_log['total']
stats.save()
# get different (distinct) user/points involved
users_points = Points.objects \
.filter(pk__gt=last_points_pk, pk__lte=newest_points_pk) \
.values('user').distinct()
total_users = users_points.count()
print('%d different user/points to process.' % total_users)
for user_points in users_points:
user = User.objects.get(pk=user_points['user'])
points, created = UserPointsSummary.objects.get_or_create(user=user)
points.update_points(last_points_pk=last_points_pk, newest_points_pk=newest_points_pk)
# update last tracker and points PKs with the last one processed
SettingProperties.objects.update_or_create(key='last_tracker_pk', defaults={"int_value": newest_tracker_pk})
SettingProperties.objects.update_or_create(key='last_points_pk', defaults={"int_value": newest_points_pk})
SettingProperties.delete_key('oppia_summary_cron_lock')