def test_delete_int(self): key = "testkey1" value = 123 SettingProperties.set_int(key, value) SettingProperties.delete_key(key) retreived_value = SettingProperties.get_int(key, 0) self.assertEqual(0, retreived_value)
def test_delete_string(self): key = "testkey1" value = "testval" SettingProperties.set_string(key, value) SettingProperties.delete_key(key) retreived_value = SettingProperties.get_string(key, "default") self.assertEqual("default", retreived_value)
def handle(self, *args, **options): # check if cron already running prop, created = SettingProperties.objects \ .get_or_create(key='oppia_summary_cron_lock', int_value=1) if not created: self.stdout.write("Oppia summary cron is already running") return try: SettingProperties.objects.get(key='oppia_cron_lock') self.stdout.write("Oppia cron is already running") SettingProperties.delete_key('oppia_summary_cron_lock') return except SettingProperties.DoesNotExist: # do nothing pass if options['fromstart']: self.update_summaries(0, 0, options['fromstart']) else: # get last tracker and points PKs processed last_tracker_pk = SettingProperties \ .get_property('last_tracker_pk', 0) last_points_pk = SettingProperties \ .get_property('last_points_pk', 0) self.update_summaries(last_tracker_pk, last_points_pk)
def test_summary_cron_locked_cron_locked(self): # set locked SettingProperties.set_int('oppia_cron_lock', 1) lock = SettingProperties.get_int('oppia_cron_lock', 0) self.assertEqual(lock, 1) SettingProperties.set_int('oppia_summary_cron_lock', 1) lock = SettingProperties.get_int('oppia_summary_cron_lock', 0) self.assertEqual(lock, 1) call_command('update_summaries', stdout=StringIO()) # check new details on pks # cron is locked so nothing should happen tracker_id = SettingProperties.get_int('last_tracker_pk', 0) self.assertEqual(tracker_id, 0) # unlock SettingProperties.delete_key('oppia_summary_cron_lock') SettingProperties.delete_key('oppia_cron_lock') # check unlocked again lock = SettingProperties.get_int('oppia_summary_cron_lock', 999) self.assertEqual(lock, 999) lock = SettingProperties.get_int('oppia_cron_lock', 999) self.assertEqual(lock, 999)
def handle(self, *args, **options): if options['hours']: hours = options['hours'] else: hours = 0 # check if cron already running prop, created = SettingProperties.objects.get_or_create(key='oppia_cron_lock', int_value=1) if not created: self.stdout.write("Oppia cron is already running") return now = time.time() path = os.path.join(settings.COURSE_UPLOAD_DIR, "temp") if os.path.exists(path): self.stdout.write('Cleaning up: ' + path) for f in os.listdir(path): f = os.path.join(path, f) if os.stat(f).st_mtime < now - 3600 * 6: self.stdout.write("deleting: {file}".format(file=f)) if os.path.isfile(f): os.remove(f) else: self.stdout.write('{path} does not exist. Don\'t need to clean it'.format(path=path)) from oppia.awards import courses_completed courses_completed(int(hours)) # create and new media images call_command('generate_media_images') SettingProperties.set_string('oppia_cron_last_run', timezone.now()) SettingProperties.delete_key('oppia_cron_lock')
def handle(self, *args, **options): if options['hours']: hours=options['hours'] else: hours=0 #check if cron already running prop, created = SettingProperties.objects.get_or_create(key='oppia_cron_lock',int_value=1) if not created: print("Oppia cron is already running") return now = time.time() path = os.path.join(settings.COURSE_UPLOAD_DIR, "temp") if os.path.exists(path): print('Cleaning up: ' + path) for f in os.listdir(path): f = os.path.join(path, f) if os.stat(f).st_mtime < now - 3600 * 6: print("deleting: {file}".format(file=f)) if os.path.isfile(f): os.remove(f) else: print('{path} does not exist. Don\'t need to clean it'.format(path=path)) from oppia.awards import courses_completed courses_completed(int(hours)) # create and new media images call_command('generate_media_images') SettingProperties.set_string('oppia_cron_last_run', timezone.now()) SettingProperties.delete_key('oppia_cron_lock')
def handle(self, *args, **options): if options['hours']: hours = options['hours'] else: hours = 0 # check if cron already running prop, created = SettingProperties.objects \ .get_or_create(key='oppia_cron_lock', int_value=1) if not created: self.stdout.write("Oppia cron is already running") return now = time.time() path = os.path.join(settings.COURSE_UPLOAD_DIR, "temp") if os.path.exists(path): self.stdout.write('Cleaning up: ' + path) for f in os.listdir(path): f = os.path.join(path, f) if os.stat(f).st_mtime < now - 3600 * 6: self.stdout.write("deleting: {file}".format(file=f)) if os.path.isfile(f): os.remove(f) else: self.stdout \ .write('{path} does not exist. Don\'t need to clean it' .format(path=path)) from oppia.awards import courses_completed courses_completed(int(hours)) # generate pdf certificates call_command('generate_certificates') # clear any expired sessions call_command('clearsessions') # add any missing api keys for Tastypie call_command('backfill_api_keys') SettingProperties.set_string('oppia_cron_last_run', timezone.now()) SettingProperties.delete_key('oppia_cron_lock') # server registration # deliberately left until after removing cron lock in case issue with # connecting to implementation server call_command('update_server_registration')
def test_summary_cron_locked(self): # set lock not SettingProperties.set_int('oppia_summary_cron_lock', 1) lock = SettingProperties.get_int('oppia_summary_cron_lock', 0) self.assertEqual(lock, 1) update_summaries() # check new details on pks # cron is locked so nothing should happen tracker_id = SettingProperties.get_int('last_tracker_pk', 0) self.assertEqual(tracker_id, 0) #unlock SettingProperties.delete_key('oppia_summary_cron_lock') #check unlocked again lock = SettingProperties.get_int('oppia_summary_cron_lock', 999) self.assertEqual(lock, 999)
def update_summaries(self, last_tracker_pk=0, last_points_pk=0, fromstart=False): SettingProperties.set_string('oppia_summary_cron_last_run', timezone.now()) # get last tracker and points PKs to be processed # (to avoid leaving some out if new trackers arrive while processing) try: newest_tracker_pk = Tracker.objects.latest('id').id newest_points_pk = Points.objects.latest('id').id except Tracker.DoesNotExist: self.stdout.write("Tracker table is empty. Aborting cron...") SettingProperties.delete_key('oppia_summary_cron_lock') return except Points.DoesNotExist: newest_points_pk = last_points_pk print('Last tracker processed: %d\nNewest tracker: %d\n' % (last_tracker_pk, newest_tracker_pk)) if last_tracker_pk >= newest_tracker_pk: self.stdout.write('No new trackers to process. Aborting cron...') SettingProperties.delete_key('oppia_summary_cron_lock') return self.update_user_course_summary(last_tracker_pk, newest_tracker_pk, last_points_pk, newest_points_pk) self.update_course_daily_stats(last_tracker_pk, newest_tracker_pk) self.update_user_points_summary(last_points_pk, newest_points_pk) self.update_daily_active_users(last_tracker_pk, newest_tracker_pk) # update last tracker and points PKs with the last one processed SettingProperties.objects \ .update_or_create(key='last_tracker_pk', defaults={"int_value": newest_tracker_pk}) SettingProperties.objects.update_or_create( key='last_points_pk', defaults={"int_value": newest_points_pk}) SettingProperties.delete_key('oppia_summary_cron_lock')
def update_summaries(last_tracker_pk=0, last_points_pk=0): from django.contrib.auth.models import User from django.db.models import Count from oppia.models import Tracker, Points, Course from settings.models import SettingProperties from summary.models import UserCourseSummary, CourseDailyStats, UserPointsSummary #check if cron already running prop, created = SettingProperties.objects.get_or_create( key='oppia_summary_cron_lock', int_value=1) if not created: print("Oppia summary cron is already running") return SettingProperties.set_string('oppia_summary_cron_last_run', timezone.now()) # get last tracker and points PKs to be processed # (to avoid leaving some out if new trackers arrive while processing) try: newest_tracker_pk = Tracker.objects.latest('id').id newest_points_pk = Points.objects.latest('id').id except Tracker.DoesNotExist: print("Tracker table is empty. Aborting cron...") SettingProperties.delete_key('oppia_summary_cron_lock') return except Points.DoesNotExist: newest_points_pk = last_points_pk print('Last tracker processed: %d\nNewest tracker: %d\n' % (last_tracker_pk, newest_tracker_pk)) if last_tracker_pk >= newest_tracker_pk: print('No new trackers to process. Aborting cron...') SettingProperties.delete_key('oppia_summary_cron_lock') return first_tracker = (last_tracker_pk == 0) first_points = (last_points_pk == 0) # If we are calculating from the start, delete previous summary calculations if first_tracker: UserCourseSummary.objects.all().delete() CourseDailyStats.objects.all().delete() if first_points: UserPointsSummary.objects.all().delete() # get different (distinct) user/courses involved user_courses = Tracker.objects \ .filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \ .exclude(course__isnull=True) \ .values('course', 'user').distinct() total_users = user_courses.count() print('%d different user/courses to process.' % total_users) count = 1 for uc_tracker in user_courses: print('processing user/course trackers... (%d/%d)' % (count, total_users)) user = User.objects.get(pk=uc_tracker['user']) course = Course.objects.get(pk=uc_tracker['course']) user_course, created = UserCourseSummary.objects.get_or_create( course=course, user=user) user_course.update_summary(last_tracker_pk=last_tracker_pk, last_points_pk=last_points_pk, newest_tracker_pk=newest_tracker_pk, newest_points_pk=newest_points_pk) count += 1 # get different (distinct) courses/dates involved course_daily_type_logs = Tracker.objects \ .filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \ .exclude(course__isnull=True) \ .extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \ .values('course', 'day', 'month', 'year', 'type') \ .annotate(total=Count('type')) \ .order_by('day') total_logs = course_daily_type_logs.count() print('%d different courses/dates/types to process.' % total_logs) count = 0 for type_log in course_daily_type_logs: day = date(type_log['year'], type_log['month'], type_log['day']) course = Course.objects.get(pk=type_log['course']) stats, created = CourseDailyStats.objects.get_or_create( course=course, day=day, type=type_log['type']) stats.total = (0 if first_tracker else stats.total) + type_log['total'] stats.save() count += 1 print(count) # get different (distinct) search logs involved search_daily_logs = Tracker.objects \ .filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk, user__is_staff=False, type='search') \ .extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \ .values('day', 'month', 'year') \ .annotate(total=Count('id')) \ .order_by('day') print('%d different search/dates to process.' % search_daily_logs.count()) for search_log in search_daily_logs: day = date(search_log['year'], search_log['month'], search_log['day']) stats, created = CourseDailyStats.objects.get_or_create(course=None, day=day, type='search') stats.total = (0 if first_tracker else stats.total) + search_log['total'] stats.save() # get different (distinct) user/points involved users_points = Points.objects \ .filter(pk__gt=last_points_pk, pk__lte=newest_points_pk) \ .values('user').distinct() total_users = users_points.count() print('%d different user/points to process.' % total_users) for user_points in users_points: user = User.objects.get(pk=user_points['user']) points, created = UserPointsSummary.objects.get_or_create(user=user) points.update_points(last_points_pk=last_points_pk, newest_points_pk=newest_points_pk) # update last tracker and points PKs with the last one processed SettingProperties.objects.update_or_create( key='last_tracker_pk', defaults={"int_value": newest_tracker_pk}) SettingProperties.objects.update_or_create( key='last_points_pk', defaults={"int_value": newest_points_pk}) SettingProperties.delete_key('oppia_summary_cron_lock')
def update_summaries(last_tracker_pk=0, last_points_pk=0): from django.contrib.auth.models import User from django.db.models import Count from oppia.models import Tracker, Points, Course from settings.models import SettingProperties from summary.models import UserCourseSummary, CourseDailyStats, UserPointsSummary #check if cron already running prop, created = SettingProperties.objects.get_or_create(key='oppia_summary_cron_lock',int_value=1) if not created: print("Oppia summary cron is already running") return SettingProperties.set_string('oppia_summary_cron_last_run', timezone.now()) # get last tracker and points PKs to be processed # (to avoid leaving some out if new trackers arrive while processing) try: newest_tracker_pk = Tracker.objects.latest('id').id newest_points_pk = Points.objects.latest('id').id except Tracker.DoesNotExist: print("Tracker table is empty. Aborting cron...") SettingProperties.delete_key('oppia_summary_cron_lock') return except Points.DoesNotExist: newest_points_pk = last_points_pk print ('Last tracker processed: %d\nNewest tracker: %d\n' % (last_tracker_pk, newest_tracker_pk)) if last_tracker_pk >= newest_tracker_pk: print('No new trackers to process. Aborting cron...') SettingProperties.delete_key('oppia_summary_cron_lock') return first_tracker = (last_tracker_pk == 0) first_points = (last_points_pk == 0) # If we are calculating from the start, delete previous summary calculations if first_tracker: UserCourseSummary.objects.all().delete() CourseDailyStats.objects.all().delete() if first_points: UserPointsSummary.objects.all().delete() # get different (distinct) user/courses involved user_courses = Tracker.objects \ .filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \ .exclude(course__isnull=True) \ .values('course', 'user').distinct() total_users = user_courses.count() print('%d different user/courses to process.' % total_users) count = 1 for uc_tracker in user_courses: print('processing user/course trackers... (%d/%d)' % (count, total_users)) user = User.objects.get(pk=uc_tracker['user']) course = Course.objects.get(pk=uc_tracker['course']) user_course, created = UserCourseSummary.objects.get_or_create(course=course, user=user) user_course.update_summary( last_tracker_pk=last_tracker_pk, last_points_pk=last_points_pk, newest_tracker_pk=newest_tracker_pk, newest_points_pk=newest_points_pk) count += 1 # get different (distinct) courses/dates involved course_daily_type_logs = Tracker.objects \ .filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk) \ .exclude(course__isnull=True) \ .extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \ .values('course', 'day', 'month', 'year', 'type') \ .annotate(total=Count('type')) \ .order_by('day') total_logs = course_daily_type_logs.count() print('%d different courses/dates/types to process.' % total_logs) count = 0 for type_log in course_daily_type_logs: day = date(type_log['year'], type_log['month'], type_log['day']) course = Course.objects.get(pk=type_log['course']) stats, created = CourseDailyStats.objects.get_or_create(course=course, day=day, type=type_log['type']) stats.total = (0 if first_tracker else stats.total) + type_log['total'] stats.save() count += 1 print(count) # get different (distinct) search logs involved search_daily_logs = Tracker.objects \ .filter(pk__gt=last_tracker_pk, pk__lte=newest_tracker_pk, user__is_staff=False, type='search') \ .extra({'day': "day(tracker_date)", 'month': "month(tracker_date)", 'year': "year(tracker_date)"}) \ .values('day', 'month', 'year') \ .annotate(total=Count('id')) \ .order_by('day') print('%d different search/dates to process.' % search_daily_logs.count()) for search_log in search_daily_logs: day = date(search_log['year'], search_log['month'], search_log['day']) stats, created = CourseDailyStats.objects.get_or_create(course=None, day=day, type='search') stats.total = (0 if first_tracker else stats.total) + search_log['total'] stats.save() # get different (distinct) user/points involved users_points = Points.objects \ .filter(pk__gt=last_points_pk, pk__lte=newest_points_pk) \ .values('user').distinct() total_users = users_points.count() print('%d different user/points to process.' % total_users) for user_points in users_points: user = User.objects.get(pk=user_points['user']) points, created = UserPointsSummary.objects.get_or_create(user=user) points.update_points(last_points_pk=last_points_pk, newest_points_pk=newest_points_pk) # update last tracker and points PKs with the last one processed SettingProperties.objects.update_or_create(key='last_tracker_pk', defaults={"int_value": newest_tracker_pk}) SettingProperties.objects.update_or_create(key='last_points_pk', defaults={"int_value": newest_points_pk}) SettingProperties.delete_key('oppia_summary_cron_lock')