def put_thumbs(notify_buf, jpeg_dir, prefix, suffix, video_id, store_loc):
# I wish Filesystem API worked the same for local and remote, but it don't
if store_loc == 'local':
root = local_storage_root_dir()
store_path = root + "/" + prefix + "/" + suffix + "/videos/" + str(video_id) + "/jpegs"
if default_storage.exists(store_path):
infoLog(notify_buf, "Found prior directory, removing: %s" % store_path)
dirRemove(store_path)
os.mkdir(store_path)
else:
store_path = prefix + "/" + suffix + "/videos/" + str(video_id) + "/jpegs"
default_storage.delete(store_path)
# not doing write to tmp and then mv because the file storage API limitation
image_list = os.listdir(jpeg_dir)
image_list.sort()
for fname in image_list:
infoLog(notify_buf, "Uploading: %s" % fname)
local_file = open(jpeg_dir + "/" + fname, 'rb')
store_file = default_storage.open(store_path + "/" + fname, 'wb')
file_data = local_file.read()
store_file.write(file_data)
local_file.close()
store_file.close()
infoLog(notify_buf, "Uploaded: %s files" % str(len(image_list)))
def upload(notify_buf, target_dir, target_part, prefix, suffix, video_id,
video_file, store_loc):
# I wish Filesystem API worked the same for local and remote, but it don't
if store_loc == 'local':
root = local_storage_root_dir()
store_path = root + "/" + prefix + "/" + suffix + "/videos/" + str(
video_id) + "/" + target_part
if default_storage.exists(store_path):
infoLog(notify_buf,
"Found prior directory, removing: %s" % store_path)
dirRemove(store_path)
os.mkdir(store_path)
else:
store_path = prefix + "/" + suffix + "/videos/" + str(
video_id) + "/" + target_part
default_storage.delete(store_path)
statinfo = os.stat(target_dir + "/" + video_file)
infoLog(notify_buf, "Final file size: %s" % str(statinfo.st_size))
local_file = open(target_dir + "/" + video_file, 'rb')
store_file = default_storage.open(store_path + "/" + video_file, 'wb')
store_file.write(local_file.read())
local_file.close()
store_file.close()
def put_thumbs(notify_buf, jpeg_dir, prefix, suffix, video_id, store_loc):
# I wish Filesystem API worked the same for local and remote, but it don't
if store_loc == 'local':
root = local_storage_root_dir()
store_path = root + "/" + prefix + "/" + suffix + "/videos/" + str(
video_id) + "/jpegs"
if default_storage.exists(store_path):
infoLog(notify_buf,
"Found prior directory, removing: %s" % store_path)
dirRemove(store_path)
os.mkdir(store_path)
else:
store_path = prefix + "/" + suffix + "/videos/" + str(
video_id) + "/jpegs"
default_storage.delete(store_path)
# not doing write to tmp and then mv because the file storage API limitation
image_list = os.listdir(jpeg_dir)
image_list.sort()
for fname in image_list:
infoLog(notify_buf, "Uploading: %s" % fname)
local_file = open(jpeg_dir + "/" + fname, 'rb')
store_file = default_storage.open(store_path + "/" + fname, 'wb')
file_data = local_file.read()
store_file.write(file_data)
local_file.close()
store_file.close()
infoLog(notify_buf, "Uploaded: %s files" % str(len(image_list)))
def view_submissions_to_grade(request, course_prefix, course_suffix, exam_slug):
course = request.common_page_data['course']
try:
exam = Exam.objects.get(course=course, is_deleted=0, slug=exam_slug)
except Exam.DoesNotExist:
raise Http404
if exam.mode=="draft":
exam = exam.image
submitters = ExamRecord.objects.filter(exam=exam, complete=True, time_created__lt=exam.grace_period).values('student').distinct()
fname = course_prefix+"-"+course_suffix+"-"+exam_slug+"-"+datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S")+".csv"
temp_file_path = FILE_DIR+"/"+fname
outfile = open(temp_file_path,"w+")
could_not_parse = ""
for s in submitters: #yes, there is sql in a loop here. We'll optimize later
latest_sub = ExamRecord.objects.values('student__username', 'time_created', 'json_data').filter(exam=exam, time_created__lt=exam.grace_period, student=s['student']).latest('time_created')
try:
sub_obj = json.loads(latest_sub['json_data']).iteritems()
for k,v in sub_obj:
outstring = '"%s","%s","%s"\n' % (latest_sub['student__username'], k, parse_val(v))
outfile.write(outstring)
except ValueError:
could_not_parse += latest_sub['student__username']+ " " #Don't output if the latest submission was erroneous
outfile.write("\n")
#if there were items we could not parse
if could_not_parse:
#write the usernames at the beginning of the file
outfile.seek(0)
data=outfile.read()
outfile.seek(0)
outfile.truncate()
outfile.write("Could not parse data from the following users: " + could_not_parse + "\n")
outfile.write(data)
if is_storage_local():
outfile.close()
reports_dir = local_storage_root_dir() + "/" + course_prefix + "/" + course_suffix + "/reports/"
if not default_storage.exists(reports_dir):
os.mkdir(reports_dir)
copyfile(temp_file_path, reports_dir + fname)
file_url = local_file_server_root() + "/" + course_prefix + "/" + course_suffix + "/reports/" + fname
return HttpResponseRedirect(file_url)
else:
#write to S3
secure_file_storage = S3BotoStorage(bucket=AWS_SECURE_STORAGE_BUCKET_NAME, access_key=AWS_ACCESS_KEY_ID, secret_key=AWS_SECRET_ACCESS_KEY)
s3file = secure_file_storage.open("/%s/%s/reports/exams/%s" % (course_prefix, course_suffix, fname),'w')
outfile.seek(0)
s3file.write(outfile.read())
s3file.close()
outfile.close()
return HttpResponseRedirect(secure_file_storage.url_monkeypatched("/%s/%s/reports/exams/%s" % (course_prefix, course_suffix, fname), response_headers={'response-content-disposition': 'attachment'}))
def view_csv_grades(request, course_prefix, course_suffix, exam_slug):
course = request.common_page_data['course']
try:
exam = Exam.objects.get(course=course, is_deleted=0, slug=exam_slug)
except Exam.DoesNotExist:
raise Http404
if course.mode=="draft":
course = course.image
if exam.mode=="draft":
exam = exam.image
graded_students = ExamScore.objects.filter(course=course, exam=exam).values('student','student__username').distinct()
fname = course_prefix+"-"+course_suffix+"-"+exam_slug+"-grades-"+datetime.datetime.now().strftime("%Y-%m-%d-%H:%M:%S")+".csv"
temp_file_path = FILE_DIR+"/"+fname
outfile = open(temp_file_path,"w+")
could_not_parse = ""
for s in graded_students: #yes, there is sql in a loop here. We'll optimize later
score_obj = ExamScore.objects.get(course=course, exam=exam, student=s['student'])
outstring = '"%s","%s"\n' % (s['student__username'], score_obj.score)
outfile.write(outstring)
subscores = ExamScoreField.objects.filter(parent=score_obj)
for field in subscores:
outstring = '"%s","%s","%s"\n' % (s['student__username'], field.field_name, str(field.subscore))
outfile.write(outstring)
outfile.write("\n")
if is_storage_local():
outfile.close()
reports_dir = local_storage_root_dir() + "/" + course_prefix + "/" + course_suffix + "/reports/"
if not default_storage.exists(reports_dir):
os.mkdir(reports_dir)
copyfile(temp_file_path, reports_dir + fname)
file_url = local_file_server_root() + "/" + course_prefix + "/" + course_suffix + "/reports/" + fname
return HttpResponseRedirect(file_url)
else:
#write to S3
secure_file_storage = S3BotoStorage(bucket=AWS_SECURE_STORAGE_BUCKET_NAME, access_key=AWS_ACCESS_KEY_ID, secret_key=AWS_SECRET_ACCESS_KEY)
s3file = secure_file_storage.open("/%s/%s/reports/exams/%s" % (course_prefix, course_suffix, fname),'w')
outfile.seek(0)
s3file.write(outfile.read())
s3file.close()
outfile.close()
return HttpResponseRedirect(secure_file_storage.url("/%s/%s/reports/exams/%s" % (course_prefix, course_suffix, fname), response_headers={'response-content-disposition': 'attachment'}))
def duration(video):
notify_buf = []
(store_path, course_prefix, course_suffix, video_id, video_filename) = splitpath(video.file.name)
print store_path
print video_filename
cmdline = [ 'ffprobe' ]
cmdline += \
[ '-loglevel', 'error',
'-show_streams', local_storage_root_dir() + "/" + store_path,
]
infoLog(notify_buf, "GET DURATION: " + " ".join(cmdline))
result = subprocess.Popen(cmdline, stdout = subprocess.PIPE, stderr = subprocess.STDOUT)
duration_str = [x for x in result.stdout.readlines() if "duration=" in x][0]
infoLog(notify_buf, duration_str)
video.duration = int(float(duration_str.split("=")[1]))
video.save()
def handle(self, *args, **options):
if len(args) != 3:
raise CommandError("Wrong number of arguments, %d instead of 3" %
len(args))
if options['force_local'] and options['force_remote']:
raise CommandError("Can't run both local and remote")
arg_prefix = args[0]
arg_suffix = args[1]
handle = arg_prefix + "--" + arg_suffix
slug = args[2]
try:
video = Video.objects.get(course__handle=handle,
slug=slug,
mode='draft')
except MultipleObjectsReturned:
print "Found multiple videos named \"%s\"" % slug
return
except Video.DoesNotExist:
print "Video \"%s\" not found for handle \"%s\"" % (slug, handle)
return
if video.file.name == "default":
print "Video slug \"%s\" doesn't have a file listed in S3 (name=\"default\")" % slug
return
# FIXME: after confirming this works, clean these lines up.
#where = getattr(settings, 'AWS_ACCESS_KEY_ID', 'local')
#if options['force_local']:
# where='local'
#if options['force_remote']:
# where='remote'
#if where == 'local':
if (is_storage_local()
or options['force_local']) and not options['force_remote']:
media_root = local_storage_root_dir()
local_path = media_root + "/" + video.file.name
kelvinator.tasks.kelvinate(local_path, options['target_frames'],
options['notify_addr'])
print "Kelvination complete: %s" % video.file.name
else:
kelvinator.tasks.kelvinate.delay(video.file.name,
options['target_frames'],
options['notify_addr'])
print "Kelvination queued (%s): %s" % (instance, video.file.name)
def duration(video):
notify_buf = []
(store_path, course_prefix, course_suffix, video_id,
video_filename) = splitpath(video.file.name)
print store_path
print video_filename
cmdline = ['ffprobe']
cmdline += \
[ '-loglevel', 'error',
'-show_streams', local_storage_root_dir() + "/" + store_path,
]
infoLog(notify_buf, "GET DURATION: " + " ".join(cmdline))
result = subprocess.Popen(cmdline,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
duration_str = [x for x in result.stdout.readlines()
if "duration=" in x][0]
infoLog(notify_buf, duration_str)
video.duration = int(float(duration_str.split("=")[1]))
video.save()
def upload(notify_buf, target_dir, target_part, prefix, suffix, video_id, video_file, store_loc):
# I wish Filesystem API worked the same for local and remote, but it don't
if store_loc == 'local':
root = local_storage_root_dir()
store_path = root + "/" + prefix + "/" + suffix + "/videos/" + str(video_id) + "/" + target_part
if default_storage.exists(store_path):
infoLog(notify_buf, "Found prior directory, removing: %s" % store_path)
dirRemove(store_path)
os.mkdir(store_path)
else:
store_path = prefix + "/" + suffix + "/videos/" + str(video_id) + "/" + target_part
default_storage.delete(store_path)
statinfo = os.stat(target_dir + "/" + video_file)
infoLog(notify_buf, "Final file size: %s" % str(statinfo.st_size))
local_file = open(target_dir + "/" + video_file, 'rb')
store_file = default_storage.open(store_path + "/" + video_file, 'wb')
store_file.write(local_file.read())
local_file.close()
store_file.close()
def handle(self, *args, **options):
if len(args) != 3:
raise CommandError("Wrong number of arguments, %d instead of 3" % len(args))
if options['force_local'] and options['force_remote']:
raise CommandError("Can't run both local and remote")
arg_prefix=args[0]
arg_suffix=args[1]
handle=arg_prefix+"--"+arg_suffix
slug=args[2]
try:
video = Video.objects.get(course__handle=handle, slug=slug, mode='draft')
except MultipleObjectsReturned:
print "Found multiple videos named \"%s\"" % slug
return
except Video.DoesNotExist:
print "Video \"%s\" not found for handle \"%s\"" % (slug, handle)
return
if video.file.name == "default":
print "Video slug \"%s\" doesn't have a file listed in S3 (name=\"default\")" % slug
return
# FIXME: after confirming this works, clean these lines up.
#where = getattr(settings, 'AWS_ACCESS_KEY_ID', 'local')
#if options['force_local']:
# where='local'
#if options['force_remote']:
# where='remote'
#if where == 'local':
if (is_storage_local() or options['force_local']) and not options['force_remote']:
media_root = local_storage_root_dir()
local_path = media_root + "/" + video.file.name
kelvinator.tasks.kelvinate(local_path, options['target_frames'], options['notify_addr'])
print "Kelvination complete: %s" % video.file.name
else:
kelvinator.tasks.kelvinate.delay(video.file.name, options['target_frames'], options['notify_addr'])
print "Kelvination queued (%s): %s" % (instance, video.file.name)
def view_csv_grades(request, course_prefix, course_suffix, exam_slug):
course = request.common_page_data['course']
try:
exam = Exam.objects.get(course=course, is_deleted=0, slug=exam_slug)
except Exam.DoesNotExist:
raise Http404
if course.mode == "draft":
course = course.image
if exam.mode == "draft":
exam = exam.image
graded_students = ExamScore.objects.filter(
course=course, exam=exam).values('student',
'student__username').distinct()
fname = course_prefix + "-" + course_suffix + "-" + exam_slug + "-grades-" + datetime.datetime.now(
).strftime("%Y-%m-%d-%H:%M:%S") + ".csv"
temp_file_path = FILE_DIR + "/" + fname
outfile = open(temp_file_path, "w+")
could_not_parse = ""
for s in graded_students: #yes, there is sql in a loop here. We'll optimize later
score_obj = ExamScore.objects.get(course=course,
exam=exam,
student=s['student'])
outstring = '"%s","%s"\n' % (s['student__username'], score_obj.score)
outfile.write(outstring)
subscores = ExamScoreField.objects.filter(parent=score_obj)
for field in subscores:
outstring = '"%s","%s","%s"\n' % (
s['student__username'], field.field_name, str(field.subscore))
outfile.write(outstring)
outfile.write("\n")
if is_storage_local():
outfile.close()
reports_dir = local_storage_root_dir(
) + "/" + course_prefix + "/" + course_suffix + "/reports/"
if not default_storage.exists(reports_dir):
os.mkdir(reports_dir)
copyfile(temp_file_path, reports_dir + fname)
file_url = local_file_server_root(
) + "/" + course_prefix + "/" + course_suffix + "/reports/" + fname
return HttpResponseRedirect(file_url)
else:
#write to S3
secure_file_storage = S3BotoStorage(
bucket=AWS_SECURE_STORAGE_BUCKET_NAME,
access_key=AWS_ACCESS_KEY_ID,
secret_key=AWS_SECRET_ACCESS_KEY)
s3file = secure_file_storage.open(
"/%s/%s/reports/exams/%s" % (course_prefix, course_suffix, fname),
'w')
outfile.seek(0)
s3file.write(outfile.read())
s3file.close()
outfile.close()
return HttpResponseRedirect(
secure_file_storage.url("/%s/%s/reports/exams/%s" %
(course_prefix, course_suffix, fname),
response_headers={
'response-content-disposition':
'attachment'
}))
def view_submissions_to_grade(request, course_prefix, course_suffix,
exam_slug):
course = request.common_page_data['course']
try:
exam = Exam.objects.get(course=course, is_deleted=0, slug=exam_slug)
except Exam.DoesNotExist:
raise Http404
if exam.mode == "draft":
exam = exam.image
submitters = ExamRecord.objects.filter(
exam=exam, complete=True,
time_created__lt=exam.grace_period).values('student').distinct()
fname = course_prefix + "-" + course_suffix + "-" + exam_slug + "-" + datetime.datetime.now(
).strftime("%Y-%m-%d-%H:%M:%S") + ".csv"
temp_file_path = FILE_DIR + "/" + fname
outfile = open(temp_file_path, "w+")
could_not_parse = ""
for s in submitters: #yes, there is sql in a loop here. We'll optimize later
latest_sub = ExamRecord.objects.values(
'student__username', 'time_created',
'json_data').filter(exam=exam,
time_created__lt=exam.grace_period,
student=s['student']).latest('time_created')
try:
sub_obj = json.loads(latest_sub['json_data']).iteritems()
for k, v in sub_obj:
outstring = '"%s","%s","%s"\n' % (
latest_sub['student__username'], k, parse_val(v))
outfile.write(outstring)
except ValueError:
could_not_parse += latest_sub[
'student__username'] + " " #Don't output if the latest submission was erroneous
outfile.write("\n")
#if there were items we could not parse
if could_not_parse:
#write the usernames at the beginning of the file
outfile.seek(0)
data = outfile.read()
outfile.seek(0)
outfile.truncate()
outfile.write("Could not parse data from the following users: " +
could_not_parse + "\n")
outfile.write(data)
if is_storage_local():
outfile.close()
reports_dir = local_storage_root_dir(
) + "/" + course_prefix + "/" + course_suffix + "/reports/"
if not default_storage.exists(reports_dir):
os.mkdir(reports_dir)
copyfile(temp_file_path, reports_dir + fname)
file_url = local_file_server_root(
) + "/" + course_prefix + "/" + course_suffix + "/reports/" + fname
return HttpResponseRedirect(file_url)
else:
#write to S3
secure_file_storage = S3BotoStorage(
bucket=AWS_SECURE_STORAGE_BUCKET_NAME,
access_key=AWS_ACCESS_KEY_ID,
secret_key=AWS_SECRET_ACCESS_KEY)
s3file = secure_file_storage.open(
"/%s/%s/reports/exams/%s" % (course_prefix, course_suffix, fname),
'w')
outfile.seek(0)
s3file.write(outfile.read())
s3file.close()
outfile.close()
return HttpResponseRedirect(
secure_file_storage.url_monkeypatched(
"/%s/%s/reports/exams/%s" %
(course_prefix, course_suffix, fname),
response_headers={
'response-content-disposition': 'attachment'
}))
