def fill_class_summaries_from_logs(user_data_coach, students_data, dt_start_utc):
dt_end_utc = dt_start_utc + datetime.timedelta(days = 1)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(user_data_student, dt_start_utc, dt_end_utc)
query_video_logs = VideoLog.get_for_user_data_between_dts(user_data_student, dt_start_utc, dt_end_utc)
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
for i, user_data_student in enumerate(students_data):
logging.info("working on student "+str(user_data_student.user))
problem_and_video_logs = []
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs, key=lambda log: log.time_started())
if problem_and_video_logs:
LogSummary.add_or_update_entry(user_data_coach, problem_and_video_logs, ClassDailyActivitySummary, LogSummaryTypes.CLASS_DAILY_ACTIVITY, 1440)
def from_problem_logs(self):
problem_log_query = ProblemLog.all()
problem_logs = problem_log_query.fetch(1000)
problem_logs.sort(key=lambda log: log.time_taken)
grouped = dict((k, sum(1 for i in v)) for (k, v) in groupby(problem_logs, key=lambda log: log.time_taken))
return self.render_hist(grouped)
def get_classtime_table(self, student_emails, dt_start_utc):
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days=1)
column = 0
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
for student_email in student_emails:
student = users.User(email=student_email)
problem_logs = ProblemLog.get_for_user_between_dts(
student, self.dt_to_utc(dt_start_ctz),
self.dt_to_utc(dt_end_ctz))
video_logs = VideoLog.get_for_user_between_dts(
student, self.dt_to_utc(dt_start_ctz),
self.dt_to_utc(dt_end_ctz))
problem_and_video_logs = []
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs,
key=lambda log: log.time_started())
chunk_current = None
for activity in problem_and_video_logs:
if chunk_current is not None and self.dt_to_ctz(
activity.time_started()) > (chunk_current.end +
self.chunk_delta):
classtime_table.drop_into_column(chunk_current, column)
chunk_current.description()
chunk_current = None
if chunk_current is None:
chunk_current = ClassTimeChunk()
chunk_current.student = student
chunk_current.start = self.dt_to_ctz(
activity.time_started())
chunk_current.end = self.dt_to_ctz(activity.time_ended())
chunk_current.activities.append(activity)
chunk_current.end = min(self.dt_to_ctz(activity.time_ended()),
dt_end_ctz)
if chunk_current is not None:
classtime_table.drop_into_column(chunk_current, column)
chunk_current.description()
column += 1
classtime_table.balance()
return classtime_table
def get_classtime_table(self, student_emails, dt_start_utc):
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days = 1)
column = 0
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
for student_email in student_emails:
student = users.User(email=student_email)
problem_logs = ProblemLog.get_for_user_between_dts(student, self.dt_to_utc(dt_start_ctz), self.dt_to_utc(dt_end_ctz))
video_logs = VideoLog.get_for_user_between_dts(student, self.dt_to_utc(dt_start_ctz), self.dt_to_utc(dt_end_ctz))
problem_and_video_logs = []
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs, key=lambda log: log.time_started())
chunk_current = None
for activity in problem_and_video_logs:
if chunk_current is not None and self.dt_to_ctz(activity.time_started()) > (chunk_current.end + self.chunk_delta):
classtime_table.drop_into_column(chunk_current, column)
chunk_current.description()
chunk_current = None
if chunk_current is None:
chunk_current = ClassTimeChunk()
chunk_current.student = student
chunk_current.start = self.dt_to_ctz(activity.time_started())
chunk_current.end = self.dt_to_ctz(activity.time_ended())
chunk_current.activities.append(activity)
chunk_current.end = min(self.dt_to_ctz(activity.time_ended()), dt_end_ctz)
if chunk_current is not None:
classtime_table.drop_into_column(chunk_current, column)
chunk_current.description()
column += 1
classtime_table.balance()
return classtime_table
def fancy_stats_deferred(exid, start_dt, end_dt, cursor, uid, i):
key_name = ExerciseStatisticShard.make_key(exid, start_dt, end_dt, cursor)
if cursor and ExerciseStatisticShard.get_by_key_name(key_name):
# We've already run, die.
return
query = ProblemLog.all()
query.filter('exercise =', exid)
query.filter('correct =', True)
query.filter('time_done >=', start_dt)
query.filter('time_done <', end_dt)
query.order('-time_done')
if cursor:
query.with_cursor(cursor)
problem_logs = query.fetch(1000)
if len(problem_logs) > 0:
logging.info("processing %d logs for %s" % (len(problem_logs), exid))
stats = fancy_stats_from_logs(problem_logs)
pickled = pickle.dumps(stats, 2)
shard = ExerciseStatisticShard(
key_name = key_name,
exid = exid,
start_dt = start_dt,
end_dt = end_dt,
cursor = cursor,
blob_val = pickled)
shard.put()
enqueue_task(exid, start_dt, end_dt, query.cursor(), uid, i+1)
else:
# No more problem logs left to process
logging.info("Summing all stats for %s", exid)
all_stats = fancy_stats_shard_reducer(exid, start_dt, end_dt)
model = ExerciseStatistic(
key_name = ExerciseStatistic.make_key(exid, start_dt, end_dt),
exid = exid,
start_dt = start_dt,
end_dt = end_dt,
blob_val = pickle.dumps(all_stats, 2),
log_count = all_stats['log_count'])
model.put()
logging.info("done processing %d logs for %s", all_stats['log_count'], exid)
def fill_class_summaries_from_logs(user_data_coach, students_data,
dt_start_utc):
dt_end_utc = dt_start_utc + datetime.timedelta(days=1)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(
user_data_student, dt_start_utc, dt_end_utc)
query_video_logs = VideoLog.get_for_user_data_between_dts(
user_data_student, dt_start_utc, dt_end_utc)
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
for i, user_data_student in enumerate(students_data):
logging.info("working on student " + str(user_data_student.user))
problem_and_video_logs = []
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs,
key=lambda log: log.time_started())
if problem_and_video_logs:
LogSummary.add_or_update_entry(
user_data_coach, problem_and_video_logs,
ClassDailyActivitySummary,
LogSummaryTypes.CLASS_DAILY_ACTIVITY, 1440)
def get(self):
user = util.get_current_user()
student = user
student_email = self.request.get('student_email')
if user:
pass
else:
student = users.User(email=student_email)
user = student
if student_email and student_email != user.email():
#logging.info("user is a coach trying to look at data for student")
student = users.User(email=student_email)
user_data = UserData.get_or_insert_for(student)
else:
#logging.info("user is a student looking at their own data")
user_data = UserData.get_or_insert_for(user)
user_data_dict = {
'email': user_data.user.email(),
'moderator': user_data.moderator,
'joined': self.datetime_to_str(user_data.joined),
'last_login': self.datetime_to_str(user_data.last_login),
'proficient_exercises': user_data.proficient_exercises,
'all_proficient_exercises': user_data.all_proficient_exercises,
'suggested_exercises': user_data.suggested_exercises,
'assigned_exercises': user_data.assigned_exercises,
'need_to_reassess': user_data.need_to_reassess,
'points': user_data.points,
'coaches': user_data.coaches,
}
user_exercises = []
for ue in UserExercise.all().filter('user ='******'exercise': ue.exercise,
'streak': ue.streak,
'longest_streak': ue.longest_streak,
'first_done': self.datetime_to_str(ue.first_done),
'last_done': self.datetime_to_str(ue.last_done),
'total_done': ue.total_done,
'last_review': self.datetime_to_str(ue.last_review),
'review_interval_secs': ue.review_interval_secs,
'proficient_date': self.datetime_to_str(ue.proficient_date),
}
user_exercises.append(ue_dict)
user_videos = []
for uv in UserVideo.all().filter('user ='******'video': uv.video.youtube_id,
'seconds_watched': uv.seconds_watched,
'last_second_watched': uv.last_second_watched,
'last_watched': self.datetime_to_str(uv.last_watched),
}
user_videos.append(uv_dict)
problems = []
for problem in ProblemLog.all().filter('user ='******'exercise': problem.exercise,
'correct': problem.correct,
'time_done': self.datetime_to_str(problem.time_done),
'time_taken': problem.time_taken,
'problem_number': problem.problem_number,
'hint_used': problem.hint_used
}
problems.append(problem_dict)
export_dict = {'UserData': user_data_dict,
'UserExercise': user_exercises,
'ProblemLog': problems,
'UserVideo': user_videos}
self.response.out.write(json.dumps(export_dict, indent=4))
def post(self):
user = util.get_current_user()
student = user
if not user:
self.response.out.write("please login first")
elif App.is_dev_server:
student_email = self.request.get('student_email')
if student_email and student_email != user.email():
#logging.info("user is a coach trying to look at data for student")
student = users.User(email=student_email)
user_data = UserData.get_or_insert_for(student)
if users.is_current_user_admin():
pass # allow admin to import to anyone's account
elif user.email() not in user_data.coaches and user.email().lower() not in user_data.coaches:
raise Exception('Student '+ student_email + ' does not have you as their coach')
else:
#logging.info("user is a student looking at their own data")
user_data = UserData.get_or_insert_for(user)
file_contents = self.request.POST.get('userdata').file.read()
import_dict = json.loads(file_contents)
user_data_dict = import_dict['UserData']
user_exercises = import_dict['UserExercise']
problems_unsorted = import_dict['ProblemLog']
user_videos = import_dict['UserVideo']
user_data.moderator = user_data_dict['moderator']
user_data.joined = self.datetime_from_str(user_data_dict['joined'])
user_data.last_login = self.datetime_from_str(user_data_dict['last_login'])
user_data.proficient_exercises = user_data_dict['proficient_exercises']
user_data.all_proficient_exercises = user_data_dict['all_proficient_exercises']
user_data.suggested_exercises = user_data_dict['suggested_exercises']
user_data.assigned_exercises = user_data_dict['assigned_exercises']
user_data.need_to_reassess = user_data_dict['need_to_reassess']
user_data.points = user_data_dict['points']
user_data.coaches = user_data_dict['coaches']
user_data.put()
proficient_dates = {}
correct_in_a_row = {}
for problem in ProblemLog.all().filter('user ='******'time_done']))
for problem in problems:
problem_log = ProblemLog()
problem_log.user = student
problem_log.exercise = problem['exercise']
problem_log.time_done = self.datetime_from_str(problem['time_done'])
problem_log.correct = problem['correct']
if problem_log.correct:
if problem_log.exercise not in correct_in_a_row:
correct_in_a_row[problem_log.exercise] = 1
else:
correct_in_a_row[problem_log.exercise] += 1
if not problem_log.exercise in proficient_dates and correct_in_a_row[problem_log.exercise] == 10:
proficient_dates[problem_log.exercise] = problem_log.time_done
#for coach in user_data.coaches:
# class_data = coaches.Class.get_or_insert(coach, coach=coach)
# class_data.compute_stats(user_data, problem_log.time_done.date())
problem_log.time_taken = problem['time_taken']
if problem.has_key('problem_number'):
problem_log.problem_number = problem['problem_number']
if problem.has_key('hint_used'):
problem_log.hint_used = problem['hint_used']
problem_log.put()
for user_exercise in UserExercise.all().filter('user ='******'exercise']
user_exercise.parent = user_data
user_exercise.user = student
user_exercise.exercise = ue['exercise']
user_exercise.streak = ue['streak']
user_exercise.longest_streak = ue['longest_streak']
user_exercise.first_done = self.datetime_from_str(ue['first_done'])
user_exercise.last_done = self.datetime_from_str(ue['last_done'])
user_exercise.total_done = ue['total_done']
last_review = self.datetime_from_str(ue['last_review'])
if last_review:
user_exercise.last_review = last_review
user_exercise.review_interval_secs = ue['review_interval_secs']
if user_exercise.exercise in proficient_dates:
user_exercise.proficient_date = proficient_dates[user_exercise.exercise]
else:
user_exercise.proficient_date = self.datetime_from_str(ue['proficient_date'])
user_exercise.put()
for user_video in UserVideo.all().filter('user ='******'/individualreport?student_email='+student.email())
else:
self.response.out.write("import is not supported on the live site")
def get_classtime_table_old(self, students_data, dt_start_utc):
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days=1)
column = 0
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(
user_data_student, self.dt_to_utc(dt_start_ctz),
self.dt_to_utc(dt_end_ctz))
query_video_logs = VideoLog.get_for_user_data_between_dts(
user_data_student, self.dt_to_utc(dt_start_ctz),
self.dt_to_utc(dt_end_ctz))
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
rows = 0
chunks = 0
for i, user_data_student in enumerate(students_data):
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
problem_and_video_logs = []
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs,
key=lambda log: log.time_started())
rows += len(problem_and_video_logs)
chunk_current = None
for activity in problem_and_video_logs:
if chunk_current is not None and self.dt_to_ctz(
activity.time_started()) > (chunk_current.end +
self.chunk_delta):
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
chunk_current = None
if chunk_current is None:
chunk_current = ClassTimeChunk()
chunk_current.user_data_student = user_data_student
chunk_current.start = self.dt_to_ctz(
activity.time_started())
chunk_current.end = self.dt_to_ctz(activity.time_ended())
chunk_current.activities.append(activity)
chunk_current.end = min(self.dt_to_ctz(activity.time_ended()),
dt_end_ctz)
if chunk_current is not None:
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
column += 1
logging.info("old rows=" + str(rows) + ", old chunks=" + str(chunks))
classtime_table.balance()
return classtime_table
def get(self):
user = util.get_current_user()
student = user
if user:
student_email = self.request.get('student_email')
if student_email and student_email != user.email():
#logging.info("user is a coach trying to look at data for student")
student = users.User(email=student_email)
user_data = UserData.get_or_insert_for(student)
if users.is_current_user_admin():
pass # allow admin to export anyone's data
elif user.email() not in user_data.coaches and user.email(
).lower() not in user_data.coaches:
raise Exception('Student ' + student_email +
' does not have you as their coach')
else:
#logging.info("user is a student looking at their own data")
user_data = UserData.get_or_insert_for(user)
user_data_dict = {
'email': user_data.user.email(),
'moderator': user_data.moderator,
'joined': self.datetime_to_str(user_data.joined),
'last_login': self.datetime_to_str(user_data.last_login),
'proficient_exercises': user_data.proficient_exercises,
'all_proficient_exercises': user_data.all_proficient_exercises,
'suggested_exercises': user_data.suggested_exercises,
'assigned_exercises': user_data.assigned_exercises,
'need_to_reassess': user_data.need_to_reassess,
'points': user_data.points,
'coaches': user_data.coaches,
}
user_exercises = []
for ue in UserExercise.all().filter('user ='******'exercise': ue.exercise,
'streak': ue.streak,
'longest_streak': ue.longest_streak,
'first_done': self.datetime_to_str(ue.first_done),
'last_done': self.datetime_to_str(ue.last_done),
'total_done': ue.total_done,
'last_review': self.datetime_to_str(ue.last_review),
'review_interval_secs': ue.review_interval_secs,
'proficient_date':
self.datetime_to_str(ue.proficient_date),
}
user_exercises.append(ue_dict)
user_videos = []
for uv in UserVideo.all().filter('user ='******'video': uv.video.youtube_id,
'seconds_watched': uv.seconds_watched,
'last_second_watched': uv.last_second_watched,
'last_watched': self.datetime_to_str(uv.last_watched),
}
user_videos.append(uv_dict)
problems = []
for problem in ProblemLog.all().filter('user ='******'exercise': problem.exercise,
'correct': problem.correct,
'time_done': self.datetime_to_str(problem.time_done),
'time_taken': problem.time_taken,
'problem_number': problem.problem_number,
'hint_used': problem.hint_used
}
problems.append(problem_dict)
export_dict = {
'UserData': user_data_dict,
'UserExercise': user_exercises,
'ProblemLog': problems,
'UserVideo': user_videos
}
self.response.out.write(json.dumps(export_dict, indent=4))
else:
self.redirect(util.create_login_url(self.request.uri))
def post(self):
user = util.get_current_user()
student = user
if not user:
self.response.out.write("please login first")
elif App.is_dev_server:
student_email = self.request.get('student_email')
if student_email and student_email != user.email():
#logging.info("user is a coach trying to look at data for student")
student = users.User(email=student_email)
user_data = UserData.get_or_insert_for(student)
if users.is_current_user_admin():
pass # allow admin to import to anyone's account
elif user.email() not in user_data.coaches and user.email(
).lower() not in user_data.coaches:
raise Exception('Student ' + student_email +
' does not have you as their coach')
else:
#logging.info("user is a student looking at their own data")
user_data = UserData.get_or_insert_for(user)
file_contents = self.request.POST.get('userdata').file.read()
import_dict = json.loads(file_contents)
user_data_dict = import_dict['UserData']
user_exercises = import_dict['UserExercise']
problems_unsorted = import_dict['ProblemLog']
user_videos = import_dict['UserVideo']
user_data.moderator = user_data_dict['moderator']
user_data.joined = self.datetime_from_str(user_data_dict['joined'])
user_data.last_login = self.datetime_from_str(
user_data_dict['last_login'])
user_data.proficient_exercises = user_data_dict[
'proficient_exercises']
user_data.all_proficient_exercises = user_data_dict[
'all_proficient_exercises']
user_data.suggested_exercises = user_data_dict[
'suggested_exercises']
user_data.assigned_exercises = user_data_dict['assigned_exercises']
user_data.need_to_reassess = user_data_dict['need_to_reassess']
user_data.points = user_data_dict['points']
user_data.coaches = user_data_dict['coaches']
user_data.put()
proficient_dates = {}
correct_in_a_row = {}
for problem in ProblemLog.all().filter('user ='******'time_done']))
for problem in problems:
problem_log = ProblemLog()
problem_log.user = student
problem_log.exercise = problem['exercise']
problem_log.time_done = self.datetime_from_str(
problem['time_done'])
problem_log.correct = problem['correct']
if problem_log.correct:
if problem_log.exercise not in correct_in_a_row:
correct_in_a_row[problem_log.exercise] = 1
else:
correct_in_a_row[problem_log.exercise] += 1
if not problem_log.exercise in proficient_dates and correct_in_a_row[
problem_log.exercise] == 10:
proficient_dates[
problem_log.exercise] = problem_log.time_done
#for coach in user_data.coaches:
# class_data = coaches.Class.get_or_insert(coach, coach=coach)
# class_data.compute_stats(user_data, problem_log.time_done.date())
problem_log.time_taken = problem['time_taken']
if problem.has_key('problem_number'):
problem_log.problem_number = problem['problem_number']
if problem.has_key('hint_used'):
problem_log.hint_used = problem['hint_used']
problem_log.put()
for user_exercise in UserExercise.all().filter('user ='******'exercise']
user_exercise.parent = user_data
user_exercise.user = student
user_exercise.exercise = ue['exercise']
user_exercise.streak = ue['streak']
user_exercise.longest_streak = ue['longest_streak']
user_exercise.first_done = self.datetime_from_str(
ue['first_done'])
user_exercise.last_done = self.datetime_from_str(
ue['last_done'])
user_exercise.total_done = ue['total_done']
last_review = self.datetime_from_str(ue['last_review'])
if last_review:
user_exercise.last_review = last_review
user_exercise.review_interval_secs = ue['review_interval_secs']
if user_exercise.exercise in proficient_dates:
user_exercise.proficient_date = proficient_dates[
user_exercise.exercise]
else:
user_exercise.proficient_date = self.datetime_from_str(
ue['proficient_date'])
user_exercise.put()
for user_video in UserVideo.all().filter('user ='******'/individualreport?student_email=' + student.email())
else:
self.response.out.write("import is not supported on the live site")
def get(self):
if not hasattr(secrets, 'ka_api_consumer_key') or \
not hasattr(secrets, 'ka_api_consumer_secret') or \
not hasattr(secrets_dev, 'ka_api_token_key') or \
not hasattr(secrets_dev, 'ka_api_token_secret'):
return self.redirect("/")
self.setup_oauth()
self.email = self.request_string("email")
if not self.email:
raise "Must supply email for user to import"
params = copy.copy(self._default_kinds)
params.update(self.request.params)
# get proper user from addition 1 userexercise
user_id_json = json.loads(
self.api("/api/v1/user/exercises/addition_1"))
user = users.User(user_id_json['user'])
# UserData
user_data_json_raw = self.api("/api/v1/user")
user_data = UserData.from_json(json.loads(user_data_json_raw),
user=user)
self.output('user_data', user_data, user_data_json_raw)
user_data.put()
if 'UserVideo' in params:
user_videos_json = json.loads(self.api("/api/v1/user/videos"))
user_videos = []
for user_video_json in user_videos_json[:params['UserVideo']]:
user_video = UserVideo.from_json(user_video_json,
user_data=user_data)
user_videos.append(user_video)
self.output('user_video', user_video, jsonify(user_video_json))
video_logs = defaultdict(list)
if 'VideoLog' in params:
for user_video in user_videos:
ytid = user_video.video.youtube_id
video_logs_json = json.loads(
self.api("/api/v1/user/videos/%s/log" % ytid))
for video_log_json in video_logs_json[:params[
'ProblemLog']]:
video_log = VideoLog.from_json(video_log_json,
user_video.video, user)
video_logs[user_video].append(video_log)
self.output("video_log", video_log,
jsonify(video_log_json))
# delete old video logs
query = VideoLog.all(keys_only=True)
query.filter('user ='******'UserExercise' in params:
user_exercises_json = json.loads(
self.api("/api/v1/user/exercises"))
user_exercises = []
for user_exercise_json in user_exercises_json[:params[
'UserExercise']]:
user_exercise = UserExercise.from_json(user_exercise_json,
user_data)
if user_exercise:
user_exercises.append(user_exercise)
self.output("user_exercise", user_exercise,
jsonify(user_exercise_json))
problem_logs = defaultdict(list)
if 'ProblemLog' in params:
for user_exercise in user_exercises:
problem_logs_json = json.loads(
self.api("/api/v1/user/exercises/%s/log" %
user_exercise.exercise))
for problem_log_json in problem_logs_json[:params[
'ProblemLog']]:
problem_log = ProblemLog.from_json(
problem_log_json,
user_data=user_data,
exercise=user_exercise.exercise_model)
problem_logs[user_exercise].append(problem_log)
self.output("problem_log", problem_log,
jsonify(problem_log_json))
db.put(user_exercises)
for k, v in problem_logs.iteritems():
db.put(v)
if 'Goal' in params:
with AutoNowDisabled(Goal):
goals_json = json.loads(self.api("/api/v1/user/goals"))
goals = []
for goal_json in goals_json[:params['Goal']]:
goal = Goal.from_json(goal_json, user_data=user_data)
goals.append(goal)
self.output("goal", goal, jsonify(goal_json))
db.put(goals)
# need to tell the userdata that it has goals
user_data.has_current_goals = not all(
[g.completed for g in goals])
user_data.put()
def get_classtime_table_old(self, students_data, dt_start_utc):
dt_start_ctz = self.dt_to_ctz(dt_start_utc)
dt_end_ctz = dt_start_ctz + datetime.timedelta(days = 1)
column = 0
classtime_table = ClassTimeTable(dt_start_ctz, dt_end_ctz)
# Asynchronously grab all student data at once
async_queries = []
for user_data_student in students_data:
query_problem_logs = ProblemLog.get_for_user_data_between_dts(user_data_student, self.dt_to_utc(dt_start_ctz), self.dt_to_utc(dt_end_ctz))
query_video_logs = VideoLog.get_for_user_data_between_dts(user_data_student, self.dt_to_utc(dt_start_ctz), self.dt_to_utc(dt_end_ctz))
async_queries.append(query_problem_logs)
async_queries.append(query_video_logs)
# Wait for all queries to finish
results = util.async_queries(async_queries, limit=10000)
rows = 0
chunks = 0
for i, user_data_student in enumerate(students_data):
problem_logs = results[i * 2].get_result()
video_logs = results[i * 2 + 1].get_result()
problem_and_video_logs = []
for problem_log in problem_logs:
problem_and_video_logs.append(problem_log)
for video_log in video_logs:
problem_and_video_logs.append(video_log)
problem_and_video_logs = sorted(problem_and_video_logs, key=lambda log: log.time_started())
rows += len(problem_and_video_logs)
chunk_current = None
for activity in problem_and_video_logs:
if chunk_current is not None and self.dt_to_ctz(activity.time_started()) > (chunk_current.end + self.chunk_delta):
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
chunk_current = None
if chunk_current is None:
chunk_current = ClassTimeChunk()
chunk_current.user_data_student = user_data_student
chunk_current.start = self.dt_to_ctz(activity.time_started())
chunk_current.end = self.dt_to_ctz(activity.time_ended())
chunk_current.activities.append(activity)
chunk_current.end = min(self.dt_to_ctz(activity.time_ended()), dt_end_ctz)
if chunk_current is not None:
chunks += 1
classtime_table.drop_into_column_old(chunk_current, column)
chunk_current.description()
column += 1
logging.info("old rows="+str(rows)+", old chunks="+str(chunks))
classtime_table.balance()
return classtime_table
def get(self):
if not hasattr(secrets, 'ka_api_consumer_key') or \
not hasattr(secrets, 'ka_api_consumer_secret') or \
not hasattr(secrets_dev, 'ka_api_token_key') or \
not hasattr(secrets_dev, 'ka_api_token_secret'):
return self.redirect("/")
self.setup_oauth()
self.email = self.request_string("email")
if not self.email:
raise "Must supply email for user to import"
params = copy.copy(self._default_kinds)
params.update(self.request.params)
# get proper user from addition 1 userexercise
user_id_json = json.loads(self.api("/api/v1/user/exercises/addition_1"))
user = users.User(user_id_json['user'])
# UserData
user_data_json_raw = self.api("/api/v1/user")
user_data = UserData.from_json(json.loads(user_data_json_raw), user=user)
self.output('user_data', user_data, user_data_json_raw)
user_data.put()
if 'UserVideo' in params:
user_videos_json = json.loads(self.api("/api/v1/user/videos"))
user_videos = []
for user_video_json in user_videos_json[:params['UserVideo']]:
user_video = UserVideo.from_json(user_video_json, user_data=user_data)
user_videos.append(user_video)
self.output('user_video', user_video, jsonify(user_video_json))
video_logs = defaultdict(list)
if 'VideoLog' in params:
for user_video in user_videos:
ytid = user_video.video.youtube_id
video_logs_json = json.loads(
self.api("/api/v1/user/videos/%s/log" % ytid))
for video_log_json in video_logs_json[:params['ProblemLog']]:
video_log = VideoLog.from_json(video_log_json, user_video.video, user)
video_logs[user_video].append(video_log)
self.output("video_log", video_log, jsonify(video_log_json))
# delete old video logs
query = VideoLog.all(keys_only=True)
query.filter('user ='******'UserExercise' in params:
user_exercises_json = json.loads(self.api("/api/v1/user/exercises"))
user_exercises = []
for user_exercise_json in user_exercises_json[:params['UserExercise']]:
user_exercise = UserExercise.from_json(user_exercise_json, user_data)
if user_exercise:
user_exercises.append(user_exercise)
self.output("user_exercise", user_exercise, jsonify(user_exercise_json))
problem_logs = defaultdict(list)
if 'ProblemLog' in params:
for user_exercise in user_exercises:
problem_logs_json = json.loads(self.api(
"/api/v1/user/exercises/%s/log" % user_exercise.exercise))
for problem_log_json in problem_logs_json[:params['ProblemLog']]:
problem_log = ProblemLog.from_json(problem_log_json,
user_data=user_data,
exercise=user_exercise.exercise_model)
problem_logs[user_exercise].append(problem_log)
self.output("problem_log", problem_log,
jsonify(problem_log_json))
db.put(user_exercises)
for k, v in problem_logs.iteritems():
db.put(v)
if 'Goal' in params:
with AutoNowDisabled(Goal):
goals_json = json.loads(self.api("/api/v1/user/goals"))
goals = []
for goal_json in goals_json[:params['Goal']]:
goal = Goal.from_json(goal_json, user_data=user_data)
goals.append(goal)
self.output("goal", goal, jsonify(goal_json))
db.put(goals)
# need to tell the userdata that it has goals
user_data.has_current_goals = not all([g.completed for g in goals])
user_data.put()