def upload():
filenames = tools.get_files(request.files)
if filenames:
merged = tools.merge_files(filenames)
uploaded = tools.serve_file(merged)
os.remove(merged)
return uploaded
def combine_pdf_pages(self, selected):
files = tools.get_files(selected)
if files:
file_in = files[0]
filename, filext = os.path.splitext(file_in)
file_out = filename + '_combined' + filext
cd = CombineDialog(_('Combine pages'), file_out)
if cd.run() == Gtk.ResponseType.ACCEPT:
size = cd.get_size()
if cd.is_vertical():
width = size[0]
height = size[1]
else:
width = size[1]
height = size[0]
filas = cd.get_rows()
columnas = cd.get_columns()
byrows = cd.is_sort_by_rows()
margen = cd.get_margin()
file_out = cd.get_file_out()
cd.destroy()
if file_out:
dialog = Progreso(_('Convert pdfs to png'), None, 1)
diboo = DoitInBackgroundOnlyOne(
pdfapi.combine, file_in, file_out, filas, columnas,
width, height, margen, byrows)
diboo.connect('done', dialog.increase)
diboo.start()
dialog.run()
def rotate_or_flip(self, selected):
files = tools.get_files(selected)
if len(files) > 0:
file0 = files[0]
fd = FlipDialog(_('Rotate files'), file0)
degrees = 0
if fd.run() == Gtk.ResponseType.ACCEPT:
fd.hide()
if fd.rbutton1.get_active():
rotate = ROTATE_000
elif fd.rbutton2.get_active():
rotate = ROTATE_090
elif fd.rbutton3.get_active():
rotate = ROTATE_180
elif fd.rbutton4.get_active():
rotate = ROTATE_270
flip_vertical = fd.switch1.get_active()
flip_horizontal = fd.switch2.get_active()
overwrite = fd.rbutton0.get_active()
dialog = Progreso(_('Rotate pdf files'), None, len(files))
diboo = DoitInBackgroundWithArgs(
cairoapi.rotate_and_flip_pages, files, rotate,
flip_vertical, flip_horizontal, overwrite)
diboo.connect('done', dialog.increase)
diboo.connect('todo', dialog.set_todo_label)
dialog.connect('i-want-stop', diboo.stop_it)
diboo.start()
dialog.run()
fd.destroy()
def rotate_some_pages(self, selected):
files = tools.get_files(selected)
if files:
file0 = files[0]
filename, filext = os.path.splitext(file0)
file_out = filename + '_rotated.pdf'
last_page = cairoapi.get_num_of_pages(file0)
spd = SelectPagesRotateDialog(_('Rotate some pages'), last_page,
file_out)
if spd.run() == Gtk.ResponseType.ACCEPT:
ranges = tools.get_ranges(spd.entry1.get_text())
if spd.rbutton1.get_active():
degrees = 270
elif spd.rbutton2.get_active():
degrees = 90
else:
degrees = 180
spd.destroy()
if len(ranges) > 0:
dialog = Progreso(_('Rotate some pages in pdf'), None, 1)
diboo = DoitInBackgroundOnlyOne(
pdfapi.rotate_ranges_in_pdf, file0, file_out,
degrees, ranges)
diboo.connect('done', dialog.increase)
diboo.start()
dialog.run()
else:
spd.destroy()
def resize_pdf_pages(self, selected):
files = tools.get_files(selected)
if files:
file_in = files[0]
filename, filext = os.path.splitext(file_in)
file_out = filename + '_resized' + filext
cd = ResizeDialog(_('Resize pages'), file_out)
if cd.run() == Gtk.ResponseType.ACCEPT:
size = cd.get_size()
if cd.is_vertical():
width = size[0]
height = size[1]
else:
width = size[1]
height = size[0]
file_out = cd.get_file_out()
cd.destroy()
if file_out:
dialog = Progreso(_('Convert pdfs to png'), None, 1)
diboo = DoitInBackgroundOnlyOne(
pdfapi.resize, file_in, file_out, width, height)
diboo.connect('done', dialog.increase)
diboo.start()
dialog.run()
cd.destroy()
def create_pdf_from_images(self, selected):
files = tools.get_files(selected)
if files:
file_in = files[0]
filename, filext = os.path.splitext(file_in)
file_out = filename + '_from_images.pdf'
cpfi = CreatePDFFromImagesDialog(
_('Create pdf from images'), files, file_out)
if cpfi.run() == Gtk.ResponseType.ACCEPT:
cpfi.hide()
files = cpfi.get_png_files()
if cpfi.is_vertical():
width, height = cpfi.get_size()
else:
height, width = cpfi.get_size()
margin = cpfi.get_margin()
file_out = cpfi.get_file_out()
cpfi.destroy()
if file_out:
dialog = Progreso(_('Convert pdfs to png'), None, 1)
diboo = DoitInBackgroundOnlyOne(
tools.create_from_images, file_out, files, width,
height, margin)
diboo.connect('done', dialog.increase)
diboo.start()
dialog.run()
cpfi.destroy()
def textmark(self, selected):
files = tools.get_files(selected)
if len(files) > 0:
file0 = files[0]
wd = TextmarkDialog(file0)
if wd.run() == Gtk.ResponseType.ACCEPT:
wd.hide()
text = wd.get_text()
color = wd.get_color()
font = wd.get_font()
size = wd.get_size()
hoption = wd.get_horizontal_option()
voption = wd.get_vertical_option()
horizontal_margin = wd.get_horizontal_margin()
vertical_margin = wd.get_vertical_margin()
dialog = Progreso(_('Textmark pdf files'), None, len(files))
diboo = DoitInBackgroundWithArgs(
cairoapi.add_textmark_to_all_pages, files, text, color,
font, size, hoption, voption, horizontal_margin,
vertical_margin, wd.rbutton0.get_active())
diboo.connect('done', dialog.increase)
diboo.connect('todo', dialog.set_todo_label)
dialog.connect('i-want-stop', diboo.stop_it)
diboo.start()
dialog.run()
wd.destroy()
def convert_pdf_file_to_png(self, selected):
files = tools.get_files(selected)
dialog = Progreso(_('Convert pdfs to png'), None, len(files))
dib = DoitInBackground(tools.convert_pdf_to_png, files)
dialog.connect('i-want-stop', dib.stop_it)
dib.connect('done', dialog.increase)
dib.connect('todo', dialog.set_todo_label)
dib.start()
dialog.run()
def reduce(self, selected):
files = tools.get_files(selected)
dialog = Progreso(_('Reduce pdf files size'), None, len(files))
diboo = DoitInBackground(
tools.reduce_pdf, files)
diboo.connect('done', dialog.increase)
diboo.connect('todo', dialog.set_todo_label)
dialog.connect('i-want-stop', diboo.stop_it)
diboo.start()
dialog.run()
def split_pdf_files(self, selected):
files = tools.get_files(selected)
if files:
dialog = Progreso(_('Split pdf files'), None, len(files))
diboo = DoitInBackground(cairoapi.split_pdf, files)
diboo.connect('done', dialog.increase)
diboo.connect('todo', dialog.set_todo_label)
dialog.connect('i-want-stop', diboo.stop_it)
diboo.start()
dialog.run()
def extract_text(self, selected):
files = tools.get_files(selected)
if files:
file0 = files[0]
filename, filext = os.path.splitext(file0)
file_out = filename + '.txt'
print(file_out)
file_out = tools.dialog_save_as_text(
_('Select file to save extracted text'), file_out)
if file_out:
pdfapi.extract_text(file0, file_out)
def join_pdf_files(self, selected):
files = tools.get_files(selected)
if files:
file_in = files[0]
filename, filext = os.path.splitext(file_in)
file_out = filename + '_joined_files.pdf'
jpd = JoinPdfsDialog(_('Join pdf files'), files, file_out)
if jpd.run() == Gtk.ResponseType.ACCEPT:
files = jpd.get_pdf_files()
file_out = jpd.get_file_out()
jpd.destroy()
if len(files) > 0 and file_out:
dialog = Progreso(_('Join pdf files'), None, 1)
diboo = DoitInBackgroundOnlyOne(
pdfapi.join_files, files, file_out)
diboo.connect('done', dialog.increase)
diboo.start()
dialog.run()
jpd.destroy()
def extract_some_pages(self, selected):
files = tools.get_files(selected)
if files:
file0 = files[0]
filename, filext = os.path.splitext(file0)
file_out = filename + '_extracted_pages.pdf'
last_page = cairoapi.get_num_of_pages(file0)
spd = SelectPagesDialog(_('Extract some pages'), last_page,
file_out)
if spd.run() == Gtk.ResponseType.ACCEPT:
ranges = tools.get_ranges(spd.entry1.get_text())
file_out = spd.get_file_out()
spd.destroy()
if len(ranges) > 0:
dialog = Progreso(_('Extract some pages in pdf'), None, 1)
diboo = DoitInBackgroundOnlyOne(
pdfapi.extract_ranges, file0, file_out, ranges)
diboo.connect('done', dialog.increase)
diboo.start()
dialog.run()
else:
spd.destroy()
def watermark(self, selected):
files = tools.get_files(selected)
if len(files) > 0:
file0 = files[0]
wd = WatermarkDialog(file0)
if wd.run() == Gtk.ResponseType.ACCEPT:
wd.hide()
hoption = wd.get_horizontal_option()
voption = wd.get_vertical_option()
horizontal_margin = wd.get_horizontal_margin()
vertical_margin = wd.get_vertical_margin()
zoom = float(wd.get_watermark_zoom()/100.0)
dialog = Progreso(_('Watermark pdf files'), None, len(files))
diboo = DoitInBackgroundWithArgs(
cairoapi.add_watermark_to_all_pages, files,
wd.get_image_filename(), hoption, voption,
horizontal_margin, vertical_margin, zoom,
wd.rbutton0.get_active())
diboo.connect('done', dialog.increase)
diboo.connect('todo', dialog.set_todo_label)
dialog.connect('i-want-stop', diboo.stop_it)
diboo.start()
dialog.run()
wd.destroy()
def convert_dir(dir_p, out_path):
for img_p in get_files(dir_p, suffix='png'):
convert(img_p, out_path)
def classification():
st.markdown("## **Master's Project**")
st.markdown('---')
st.markdown("# **Supervised Learning | Classification**")
st.markdown("### Choose a Dataset")
files = tools.get_files()
option = st.selectbox('Select a Teacher Answer', files['name'])
index = files['name'].index(option)
st.write('You selected:', index)
doc = files['doc'][index]
data = files['data'][index]
st.markdown("## Modeling Example")
st.markdown('**Select the below based on what you want to see:**')
doc_flag = st.checkbox('Display Question Info')
data_flag = st.checkbox('Display Prediction Data')
model_flag = st.checkbox('Display Model Info and Performance')
test_size = st.number_input('Test Size',
min_value=0.01,
max_value=0.91,
value=.75,
step=0.05)
nlp = Classification_NLP(data, doc, test_size)
st.markdown(f'Training Set Size: {len(nlp.X_train)}')
st.markdown(f'Test Set Size: {len(nlp.X_test)}')
if doc_flag:
st.markdown('## Question Info')
st.write(doc)
if data_flag:
st.markdown('## Prediction Data')
st.write(data)
st.write(pd.Series(data.columns, name='Features'))
if model_flag:
st.markdown('## Model Data')
_, accuracy = nlp.accuracy()
st.markdown(
f'### **Test Set Accuracy of Model: {round(accuracy, 3)}** ')
st.pyplot(fig=charts.plot_confusion_matrix(nlp.y_test, nlp.pred))
results = doc[['student_answer', 'label', 'prediction']]
try_it = st.checkbox('Try it Yourself!')
explore_flag = st.checkbox('Explore Data')
if try_it:
tryit(nlp)
if explore_flag:
st.markdown(f"""**Dataset Length: {len(results)}** """)
start, end = st.slider(label='Data View Select',
min_value=0,
max_value=len(nlp.doc) - 1,
value=(0, 5))
st.markdown(
f"""**Teacher Answer: {nlp.doc['teacher_answer'].values[0]}**"""
)
for i in range(int(start), int(end) + 1):
if nlp.doc.loc[i, 'label'] != nlp.doc.loc[i, 'prediction']:
st.markdown(
f"""{i}. {'Label:':>10} {str(nlp.doc.loc[i,'label'])} Pred: {str(nlp.doc.loc[i,'prediction'])} {str(nlp.doc.loc[i,'student_answer'])}"""
)
else:
st.markdown(
f"""{i}. {'Label:':>10} {str(nlp.doc.loc[i,'label'])} Pred: {str(nlp.doc.loc[i,'prediction'])} {str(nlp.doc.loc[i,'student_answer'])}"""
)
def clustering():
st.markdown("## **Master's Project**")
st.markdown('---')
st.markdown("# **Unsupervised Learning | Clustering**")
st.markdown("### Choose a Dataset")
files = tools.get_files()
option = st.selectbox('Select a Teacher Answer', files['name'])
index = files['name'].index(option)
st.write('You selected:', index)
doc = files['doc'][index]
data = files['data'][index]
st.markdown("## Modeling Example")
st.markdown('**Select the below based on what you want to see:**')
doc_flag = st.checkbox('Display Question Info')
data_flag = st.checkbox('Display Prediction Data')
chart_flag = st.checkbox('Display Reduced-Dimensionality Chart')
model_flag = st.checkbox('Display Model Info and Performance')
nlp = Clustering_NLP(data, doc)
nlp.correct_cluster_labels()
if doc_flag:
st.markdown('## Question Info')
st.write(doc)
if data_flag:
st.write(data.columns)
st.markdown('## Prediction Data')
st.write(data)
st.write(pd.Series(data.columns, name='Features'))
if chart_flag:
col1, col2 = st.beta_columns(2)
fig1, ax1 = charts.plot_pca_chart(data, doc['label'],
nlp.model.cluster_centers_)
fig2, ax2 = charts.plot_tsne_chart(data, doc['label'],
nlp.model.cluster_centers_)
with col1:
st.pyplot(fig=fig1)
with col2:
st.pyplot(fig=fig2)
if model_flag:
st.markdown('## Model Data')
st.markdown(f'### **Accuracy of Model: {round(nlp.accuracy(),3)}** ')
st.pyplot(fig=charts.plot_confusion_matrix(nlp.doc['label'],
nlp.doc.cluster))
results = doc[['student_answer', 'label', 'cluster']]
try_it = st.checkbox('Try it Yourself!')
explore_flag = st.checkbox('Explore Data')
if try_it:
tryit(nlp)
if explore_flag:
st.markdown(f"""**Dataset Length: {len(results)}** """)
start, end = st.slider(label='Data View Select',
min_value=0,
max_value=len(nlp.doc) - 1,
value=(0, 5))
st.markdown(
f"""**Teacher Answer: {nlp.doc['teacher_answer'].values[0]}**"""
)
for i in range(int(start), int(end) + 1):
st.markdown(
f"""{i}. {'Label:':>10} {str(nlp.doc.loc[i,'label'])} Pred: {str(nlp.doc.loc[i,'cluster'])} {str(nlp.doc.loc[i,'student_answer'])}"""
)
import pandas as pd
import numpy as np
import streamlit as st
import text
import base64
from ClusteringNLP import Clustering_NLP
from ClassificationNLP import Classification_NLP
from ClusterAndClassify import Cluster_and_Classify
from sklearn.metrics import accuracy_score
import charts
import tools
files = tools.get_files()
def landing():
file_ = open("./resources/comp_teach.gif", "rb")
contents = file_.read()
data_url = base64.b64encode(contents).decode("utf-8")
file_.close()
# Heading
st.markdown("## **Master's Project**")
st.markdown('---')
st.markdown("# **Automated Short Response Grading**")
st.markdown(f'## Purpose')
st.markdown(
""" Grading has to be one of the most tedious, unfun aspects of being a teacher. Having talked to former collegues, most would be willing to even pay an external party to do grading for them. Personally as a teacher, I always felt I had a duty to my students to complete grading on time, but with all of the other preparation work, it can feel overwhelming.
Recognizing that most problems of repatition are perfect for programs, I set out to see if I could create an application which would, to some extent, reduce the amount of grading a teacher had to do. Specifically, attempting to use unsupervised and supervised machine learning methods to address short answer response questions. """
#!/usr/bin/python
# -*- coding: UTF-8 -*-
import cv2
import logging
import os
import re
import shutil
import tools
from PIL import Image
import psutil
import pytesseract
import cv2
def ocr(fileName):
ref = cv2.imread(fileName)
ref = cv2.cvtColor(ref, cv2.COLOR_BGR2GRAY)
code = pytesseract.image_to_string(ref) # return unicode
print(code)
if __name__ == '__main__':
files = tools.get_files('C:\\Users\\Public\\ic-web')
for fileName in files:
print('===========', fileName)
ocr(fileName)
# print(files)
import sys
import os
import numpy as np
import cv2 as cv
import tools as tool
import time as t
args = sys.argv
folder_path = args[1]
sign_path = args[2]
dst_path = args[3]
t1 = t.time()
num_photos = len(os.listdir(os.getcwd()))
print("Processing", num_photos, "photos in", folder_path)
print("Using signature in", sign_path)
for filename in tool.get_files(folder_path):
signed = tool.sign_image(filename,sign_path,verbose=True)
signed_filename = dst_path + "/" + filename
cv.imwrite(signed_filename,signed)
#cv.imshow(filename,signed)
t2 = t.time() - t1
print(" ")
print(num_photos,"photos processed in",round(t2,3),"s")
print("Signed photos saved in ",dst_path)
def analyze(input_folder, model_dirs, test):
path, date = os.path.split(input_folder)
path, mouse = os.path.split(path)
desktop = tools.desktop_path()
analyzed_folder = os.path.join(desktop, 'ANALYZED', mouse, date)
pose2d_folder = os.path.join(analyzed_folder, 'POSE_2D')
pose2d_filter_folder = os.path.join(analyzed_folder,
'POSE_2D_FILTERED')
pose2d_video_folder = os.path.join(analyzed_folder, 'VIDEOS_2D')
dlc_collage_folder = os.path.join(analyzed_folder, 'COLLAGE_DLC')
pose2d_video_filtered_folder = os.path.join(analyzed_folder,
'VIDEOS_2D_FILTERED')
pose3d_folder = os.path.join(analyzed_folder, 'POSE_3D')
pose3d_filter_folder = os.path.join(analyzed_folder, 'POSE_3D_FILTER')
pose3d_video_folder = os.path.join(analyzed_folder, 'VIDEOS_3D')
combined_video_folder = os.path.join(analyzed_folder,
'VIDEOS_3D_COMBINED')
dict_of_avis = {
x: tools.get_files(input_folder, (x + '.' + base.RAW_VIDEO_EXT))
for x in base.CAMERA_NAMES
}
#TEMPORARY
import shutil
if os.path.exists(pose2d_video_folder):
shutil.rmtree(pose2d_video_folder)
if os.path.exists(pose2d_folder):
shutil.rmtree(pose2d_folder)
if os.path.exists(dlc_collage_folder):
shutil.rmtree(dlc_collage_folder)
if test:
for k, v in dict_of_avis.items():
dict_of_avis[k] = v[:1]
for cam, video_pns in dict_of_avis.items():
model = model_dirs[cam]
anp.pose_videos.process_peter(videos=video_pns,
model_folder=model,
out_folder=pose2d_folder,
video_type=base.RAW_VIDEO_EXT)
anp.label_videos.process_peter(scheme=scheme,
threshold=base.SCORE_THRESHOLD,
body_part_colors=colors,
body_part_sizes=sizes,
video_folder=input_folder,
pose_2d_folder=pose2d_folder,
out_folder=pose2d_video_folder,
video_type='avi')
videos0 = [
os.path.join(pose2d_video_folder, x)
for x in os.listdir(pose2d_video_folder) if 'CAM0' in x
]
videos0 = sorted(videos0)
video_tools.make_collage(videos0,
width=5,
height=4,
collage_folder=dlc_collage_folder,
savestr='CAM0')
def chunk(case_dir, n, done=None):
files = get_files(case_dir, suffix='tiff')
for i in range(n, len(files), n):
pps = files[i-n:i]
yield pps