def parse_tsv_files(input_path, output_dir):
if not os.path.exists(input_path):
sys.exit("Error: invalid input path '{}'".format(input_path))
input_files = []
if os.path.isfile(input_path):
input_files.append(input_path)
else:
for (root, dirs, files) in os.walk(input_path):
for file_x in files:
if file_x.endswith(".tsv"):
input_files.append(path_join(root,file_x))
if os.path.exists(output_dir):
if os.path.isfile(output_dir):
sys.exit("Error: Out arg must be directory.")
else:
os.makedirs(output_dir)
for tsv_filename in input_files:
coursename = tsv_filename.replace(".tsv","")
coursename = coursename.replace(input_path, "")
coursename = coursename.replace("/", "")
content = parse_course_tsv(tsv_filename)
dump_json(content, path_join(output_dir,coursename)+".json")
def parse_tsv_files(input_path, output_dir):
if not os.path.exists(input_path):
sys.exit("Error: invalid input path '{}'".format(input_path))
input_files = []
if os.path.isfile(input_path):
input_files.append(input_path)
else:
for (root, dirs, files) in os.walk(input_path):
for file_x in files:
if file_x.endswith(".tsv"):
input_files.append(path_join(root, file_x))
if os.path.exists(output_dir):
if os.path.isfile(output_dir):
sys.exit("Error: Out arg must be directory.")
else:
os.makedirs(output_dir)
for tsv_filename in input_files:
coursename = tsv_filename.replace(".tsv", "")
coursename = coursename.replace(input_path, "")
coursename = coursename.replace("/", "")
content = parse_course_tsv(tsv_filename)
dump_json(content, path_join(output_dir, coursename) + ".json")
def generate_scales(semester):
scales = OrderedDict()
scales_path = "./data/"+semester+"/outputs/scales.json"
default_scales_path = "./resources/scales.json"
if not os.path.exists(scales_path):
scales = load_json(default_scales_path)
else:
scales = load_json(scales_path)
if not scales:
scales = OrderedDict()
q = "Remove this example question - How do you rate the course in general? (Add questions like this)"
scales[q] = OrderedDict()
convert_answer_case(scales)
responses_path = "./data/"+semester+"/outputs/responses/"
for (dirpath, dirnames, filenames) in os.walk(responses_path):
for filename in filenames:
if filename.endswith(".json"):
file_path = path_join(dirpath,filename)
scales_add_course(file_path, scales)
break
default_sort_scales(scales)
try:
autofill_scales(scales)
except AutofillException:
save_prompt_exit(scales, scales_path)
dump_json(scales, scales_path)
if print_error_check(scales):
print("One or more inconsistency detected in " + scales_path)
print("You will have to edit the file manually to add/edit/remove questions.")
sys.exit(1)
def main(semester_dir):
files = []
for f in os.listdir(semester_dir+"/outputs/stats"):
if f.endswith(".json"):
files.append(f)
semester_data = OrderedDict()
for f in files:
course_name = f.replace(".json", "")
semester_data[course_name] = get_course_data(semester_dir+"/outputs/stats/"+f)
dump_json(semester_data, semester_dir+"/outputs/courses.json")
def main(semester_dir):
files = []
for f in os.listdir(semester_dir + "/outputs/stats"):
if f.endswith(".json"):
files.append(f)
semester_data = OrderedDict()
for f in files:
course_name = f.replace(".json", "")
semester_data[course_name] = get_course_data(semester_dir +
"/outputs/stats/" + f)
dump_json(semester_data, semester_dir + "/outputs/courses.json")
def course_list(url, path, filters_path):
filters = []
if filters_path != None:
with open(filters_path) as f:
filters = f.read().splitlines()
page = requests.get(args.url)
# write_page(page.content, path)
html = page.content.decode("utf-8")
courses = course_filter(course_dict(html), filters)
dump_json(courses, path)
def course_list(url, path, filters_path):
filters = []
if filters_path != None:
with open(filters_path) as f:
filters = f.read().splitlines()
page = requests.get(args.url)
# write_page(page.content, path)
html = page.content.decode("utf-8")
courses = course_filter(course_dict(html), filters)
dump_json(courses, path)
def generate_stats_file(responses_path, participation_path, output_path, scales, course):
responses = load_json(responses_path)
participation = load_json(participation_path)
stats = OrderedDict()
stats["course"] = course
stats = generate_stats(responses, participation, scales, stats)
if not stats:
print("Skipping course with 0 answers:")
print(json.dumps(course, indent=2))
return
if not stats["language"]:
print("Unable to detect language in course:")
print(json.dumps(course, indent=2))
print("This most likely means that the questions have changed and need to be added to scales.json")
sys.exit(1)
dump_json(stats, output_path)
def generate_stats_file(responses_path, participation_path, output_path,
scales, course):
responses = load_json(responses_path)
participation = load_json(participation_path)
stats = OrderedDict()
stats["course"] = course
stats = generate_stats(responses, participation, scales, stats)
if not stats:
print("Skipping course with 0 answers:")
print(json.dumps(course, indent=2))
return
if not stats["language"]:
print("Unable to detect language in course:")
print(json.dumps(course, indent=2))
print(
"This most likely means that the questions have changed and need to be added to scales.json"
)
sys.exit(1)
dump_json(stats, output_path)
def generate_scales(semester):
scales = OrderedDict()
scales_path = "./data/" + semester + "/outputs/scales.json"
default_scales_path = "./resources/scales.json"
if not os.path.exists(scales_path):
scales = load_json(default_scales_path)
else:
scales = load_json(scales_path)
if not scales:
scales = OrderedDict()
q = "Remove this example question - How do you rate the course in general? (Add questions like this)"
scales[q] = OrderedDict()
convert_answer_case(scales)
responses_path = "./data/" + semester + "/outputs/responses/"
for (dirpath, dirnames, filenames) in os.walk(responses_path):
for filename in filenames:
if filename.endswith(".json"):
file_path = path_join(dirpath, filename)
scales_add_course(file_path, scales)
break
default_sort_scales(scales)
try:
autofill_scales(scales)
except AutofillException:
save_prompt_exit(scales, scales_path)
dump_json(scales, scales_path)
if print_error_check(scales):
print("One or more inconsistency detected in " + scales_path)
print(
"You will have to edit the file manually to add/edit/remove questions."
)
sys.exit(1)
s_order = []
for i in range(start_year, stop_year):
s_order.append("V" + str(i))
s_order.append("H" + str(i))
return s_order
def get_semesters(path):
semester_order = get_semester_order(2000, 2030)
semesters = []
for root, subdirs, files in os.walk(path):
semesters = list(filter(lambda x: x != ".git", subdirs))
break
indices = [semester_order.index(x) for x in semesters]
semesters = [x for (y, x) in sorted(zip(indices, semesters))]
return semesters
if __name__ == '__main__':
semesters = get_semesters("./data")
courses = OrderedDict()
for s in semesters:
p = "./data/" + s + "/outputs/courses.json"
semester = load_json(p)
for course in semester:
if course not in courses:
courses[course] = OrderedDict()
courses[course][s] = semester[course]
dump_json(courses, "./data/courses.json")
def save_prompt_exit(scales, scales_path):
print("Do you want to save (overwrite) scales before quitting?(y/n)")
inp = yes_or_no()
if inp == "y":
dump_json(scales, scales_path)
sys.exit(1)
def save_prompt_exit(scales, scales_path):
print("Do you want to save (overwrite) scales before quitting?(y/n)")
inp = yes_or_no()
if inp == "y":
dump_json(scales, scales_path)
sys.exit(1)