def setUp(self): """Create questions for testing.""" grade_test = Grade(name="Grade Example") grade_test.save() subject_test = Subject(name="addition", grade=grade_test) subject_test.save() question1 = Question(question_content='what is 1 + 1?', answer_content='This is an addition question', subject=subject_test) question1.save()
def post(self): title = self.request.get('title') desc = self.request.get('desc') if not title: return self.render_json({'success': False, 'message':'Campo Titulo obrigatório!'}) else: subject = Subject(title=title, description=desc, owner=self.logged.key) subject.put() deferred.defer(index_subject, subject, _queue='subjectIndex') return self.render_json({'success': True, 'redirect': uri_for('classes_by_subject', subject_id=subject.key.id())})
def insert_schools_data(): university = University.objects.first() for school_name, degrees in schools_data.items(): school = School(name=school_name) school.university = university school.save() for degree_name, subjects in degrees.items(): degree = Degree(title=degree_name) degree.school = school degree.save() for subject_code, subject_title in subjects.items(): subject = Subject(code=subject_code, title=subject_title['title']) subject.degree = degree subject.save()
def search_subject(self): q = self.request.get('q', '') index = search.Index(name='subjectIndex') query = index.search(q) subjects = [Subject.get_by_id(int(result.doc_id)) for result in query.results] subjects = [sub for sub in subjects if sub != None] return self.render('app/search_result.html', subjects=subjects, search_term=q)
def add_subject_save(request): if request.method != "POST": return HttpResponse("<h2>Method Not Allowed</h2>") else: subject_name = request.POST.get("subject_name") class_id = request.POST.get("grade") class_name = Grade.objects.get(id=class_id) teacher_id = request.POST.get("teacher") teacher = CustomUser.objects.get(id=teacher_id) try: subject = Subject(subject_name=subject_name, class_id=class_name, teacher_id=teacher) subject.save() messages.success(request, "Successfully Added Subject") return HttpResponseRedirect(reverse("add_subject")) except: messages.error(request, "Failed to Add Subject") return HttpResponseRedirect(reverse("add_subject"))
def handle(self, *args, **options): df = pd.read_csv(sys.argv[3]) subjects = [ Subject( id=df['id'][row], name=df['name'][row], course=Course.objects.get(id=df['instituteid'][row]), ) for row in range(0, 41) ] subjects = Subject.objects.bulk_create(subjects) if subjects: self.stdout.write('Successfully loaded course-subjects..')
def by_subject(self, subject_id): subject = Subject.get_by_id(int(subject_id)) if not subject: return self.abort(404) alerts = [] if self.request.method == 'POST': title = self.request.get('title') desc = self.request.get('desc') if not title: alerts.append('Title field is required!') else: clazz = Class(title=title, description=desc, subject=subject.key) clazz.put() return self.render('app/class_detail.html', cls=clazz) classes = subject.get_classes() return self.render('app/classes.html', classes=classes, alerts=alerts, subject=subject)
def add_event(self): course_id = self.request.get('course', 1) start_date = self.request.get('date') start_time = self.request.get('time') text = self.request.get('text') course = Subject.get_by_id(int(course_id)) evt_date = datetime.datetime.strptime(start_date+start_time, '%Y-%m-%d%H:%M') cal_event = CalendarEvent(profile=self.logged.key, summary=text, start=evt_date) if course: cal_event.course = course.key cal_event.put() event = { 'summary': text, 'start_date': datetime.datetime.strftime(cal_event.start, '%Y-%m-%d %H:%M'), 'id': cal_event.key.id(), 'course': cal_event.course.id() if course else 1, 'course_title': cal_event.course.get().title if course else '', } return self.render_json(event)
def home(self): subjects = Subject.query().fetch() #self.logged.get_followed_courses() most_followed = [] #Subject.query().fetch() return self.render('app/dashboard.html', subjects=subjects, most_followed=most_followed)
def handle(self, *args, **options): Region.objects.all().delete() University.objects.all().delete() Specialty.objects.all().delete() Subject.objects.all().delete() Variant.objects.all().delete() raw_regions_list = [Region(yandex_id=region_id, name=region_name) for region_id, region_name in get_regions().items()] regions = Region.objects.bulk_create(raw_regions_list) specialties_rating = get_specialties_rating() raw_specialties_list = [Specialty(yandex_id=specialty_id, name=specialty_name, place=specialties_rating.get(specialty_name, None)) for specialty_id, specialty_name in get_specialties().items()] Specialty.objects.bulk_create(raw_specialties_list) raw_subjects_list = [Subject(name=subject_name) for subject_name in ['математика', 'русский язык', 'физика', 'обществознание', 'биология', 'история', 'химия', 'информатика', 'география', 'литература', 'иностранный язык']] Subject.objects.bulk_create(raw_subjects_list) for region in regions: for university_id in get_universities_id(region.yandex_id, list(get_specialties().keys())): university_info = get_info(university_id) variants = university_info['data'] university = University.objects.create(yandex_id=university_id, name=university_info[university_id], place=get_universities_rating(university_info[university_id]), region=region) for variant in variants: subjects = Subject.objects.filter(name__in=variant['subjects']) specialty = Specialty.objects.get(name=variant['specialty']) budget_plan = variant['budget_plan'] budget_passing_score = variant['budget_passing_score'] paid_plan = variant['paid_plan'] paid_passing_score = variant['paid_passing_score'] if specialty.place == None or \ budget_plan == None or \ budget_passing_score == None or \ paid_plan == None or \ paid_passing_score == None: continue var = Variant.objects.create(university=university, specialty=specialty, department=variant['department'], education_type=variant['level'], budget_plan=int(budget_plan), budget_passing_score=int(budget_passing_score), paid_plan=int(paid_plan), paid_passing_score=int(paid_passing_score), cost=variant['cost'], rating=None, applicants=[]) var.subjects.set(subjects) universities = pd.read_csv('universities_rating.csv', delimiter='\t') for index, row in universities.iterrows(): university = University.objects.get(yandex_id=row.yandex_id) if row.place == -1: university.place = None else: university.place = int(row.place) university.save() for variant in Variant.objects.all(): if variant.university.place != None: variant.rating = math.log(variant.university.place * variant.specialty.place) variant.save()
def scrap_from_lms(year_semester, limit=True): ''' Scrap(Parse) subjects data from lms. This function makes a baseline of database. Using parsed information of each subject, it searches database and update a matched subject or create a new one. If limit is true, this function reads one subject per 0.5 seconds, otherwise it reads the lms data as fast as it can. ''' ## input example: '2018s', '2019f' ## parsing input year = year_semester[0:4] semester_input = year_semester[4:5] ## we change semester 's' to '0' and 'f' to '2' for lms query ## semester_code is used for model 'Semester' semester_dic = {'s': '0', 'f': '2', 'S': '0', 'F': '2'} semester_code = {'0': 'S', '2': 'F'} semester_code_in_Korean = {'0': '년 1학기', '2': '년 2학기'} semester = semester_dic[semester_input] ## Semester check and if there is no semester, then make it try: Semester_object = Semester.objects.get(code=year + semester_code[semester]) except ObjectDoesNotExist: Semester_object = Semester(name=year + semester_code_in_Korean[semester], code=year + semester_code[semester]) Semester_object.save() try: print("* * * * New semester created! WOW! " + year + semester_code_in_Korean[semester] + " * * * *") except: print("* * * * New semester created! WOW! " + year + semester_code[semester] + " * * * *") ## variables lms_url = 'http://lms.postech.ac.kr/' ## Change department values to use whole departments' information department_page = requests.get(lms_url + 'Course.do?cmd=viewCourseAllList') department_tree = html.fromstring(department_page.content) department_list = department_tree.xpath('//span[@class="selectForm"]')[1] department_name = department_list.xpath('.//a[@href]/text()') department_code = department_list.xpath('.//a/attribute::href') ## save departments variables departments = [] departments_cleared_name = [] Department_objects = [] if limit: limit_str = '0.5 SEC' else: limit_str = 'UNLIMITTED!!!' print( "********************************** PARSING MESSAGE ***************************************" ) print("******* SEMESTER: " + year + " " + semester_code[semester] + " ************************* SPEED LIMIT: " + limit_str + " *******") print( "*********************************** PARSING START ****************************************" ) ## for each parsed department information, get 'Department' model object or create new one for i in range(len(department_code)): departments.append(department_code[i].split(',')[1].split("'")[1]) departments_cleared_name.append(clear_string(department_name[i])) try: Department_object = Department.objects.get( name=departments_cleared_name[i]) except ObjectDoesNotExist: Department_object = Department(name=departments_cleared_name[i]) Department_object.save() try: print( "****** New department " + departments_cleared_name[i] + " created ****** This should be done ONCE at the first time" ) except: print( "****** New department created ****** This should be done ONCE at the first time" ) Department_objects.append(Department_object) ## current page starts from 1 cur_page = 1 ## search. open subject lists by department for cur_dept in range(len(departments)): # To let you know where you are try: print("Parsing department: " + departments_cleared_name[cur_dept]) except: print("Parsing department #" + str(cur_dept)) while True: ## get a subject list of one department page = requests.get( lms_url + 'Course.do?cmd=viewCourseAllList&courseTermDTO.courseTermId=' + year + '09' + semester + '&searchCode1=' + departments[cur_dept] + '&curPage=' + str(cur_page)) if page.status_code != 200: cur_page += 1 continue tree = html.fromstring(page.content) ## find the href list href_list = tree.xpath('//td/a/attribute::href') ## if there is no subject it breaks if len(href_list) == 0: print("No more subjects in this list!") break ## start parsing! print("Parsing page # " + str(cur_page)) ## get one subject info and plan data from one subject data for cur_subject in href_list: info_page = requests.get(lms_url + cur_subject) if info_page.status_code != 200: continue info_tree = html.fromstring(info_page.content) ## get info table info_tr = info_tree.xpath('//table/tr') ## plan src plan_a = info_tree.xpath('//a/attribute::href')[0] ## EXAMPLE: 1-0-1 try: credit = clear_string(info_tr[8].xpath('td')[3].text) except: continue ## EXAMPLE: CSED101 code = clear_string(info_tr[4].xpath('td')[1].text) ## EXAMPLE: 햅틱스 입문 subject_name = clear_string(info_tr[2].xpath('td')[1].text) ## EXAMPLE: 전공선택 category = clear_string(info_tr[6].xpath('td')[3].text) ## EXAMPLE: 1 try: class_number = int( clear_string(info_tr[4].xpath('td')[3].text)) except ValueError: class_number = 1 ## EXAMPLE: '200' capacity_raw = clear_string(info_tr[10].xpath('td')[1].text) ## EXAMPLE: 200 capacity = int(re.search(r'\d+', capacity_raw).group()) ## load plan data plan_page = requests.get(lms_url + plan_a) if plan_page.status_code != 200: continue plan_tree = html.fromstring(plan_page.content) ## get plan table. sometimes it does not have any plan table (?) if len(plan_tree.xpath('//table')) > 0: plan_tr = plan_tree.xpath('//table') ## EXAMPLE: 윤은영 prof_name = clear_string(plan_tr[2].xpath('.//td')[0].text) ## EXAMPLE..... is the line below. time_place_raw = plan_tr[1].xpath('.//td')[5].xpath( './/text()') time_place_num = len(time_place_raw) ## time_place lines can be multiple, so we prepare arrays time_place = [] place = [] start_time = [] end_time = [] days = [] ## This time_place can be several line. So it goes with for loop. for line_number in range(time_place_num): time_place_text = clear_string( time_place_raw[line_number]) ## EXAMPLE: 1. 월 수 (09:30~10:45) | 무은재기념관 강의실 [101호] if '.' not in time_place_text: time_place.append(time_place_text) else: time_place.append(time_place_text.split('.')[1]) ## EXAMPLE: 월 수 (09:30~10:45) | 무은재기념관 강의실 [101호] ## Actually... I don't know why I wrote like this. ## Maybe there is a historical reason. (e.g. there is no time_place text yet.) if len(time_place[line_number]) < 8: time_place_num = 0 break ## EXAMPLE: 무은재기념관 강의실 [101호] time_place_or_splitted = time_place[line_number].split( '|') place_temp = time_place_or_splitted[1] ## EXAMPLE: ' 무은재기념관 강의실 [101호]' -> '무은재기념관 강의실 [101호]' place.append(place_temp[1:len(place_temp) + 1]) ## EXAMPLE: 월 수 (09:30~10:45) days_time = time_place_or_splitted[0] ## EXAMPLE: 09:30~10:45 time = days_time.split('(')[1].split(')')[0] ## EXAMPLE: 09:30 start_time.append(time.split('~')[0]) ## EXAMPLE: 10:45 end_time_raw = time.split('~')[1] if end_time_raw[0:2] == '24': end_time_raw = '23:59' end_time.append(end_time_raw) ## EXAMPLE: ['월', '수'] days.append(days_time.split('(')[0].split()) ## if there is no table (I don't know why I wrote this code in RoR lemons) ## just pass it. come back later. Don't save anything. else: continue ## Try to find category of subject. if except occurs, make one. try: if category == u"0035": category = u"기초선택" Category_object = Category.objects.get(category=category) except ObjectDoesNotExist: Category_object = Category(category=category) Category_object.save() try: print("***** New category: " + category + " added *****") except: print("***** New category added *****") ## Try to find Subject. try: ## Find subject by code, class_number, semester and name ## This should be just one Subject_object = Subject.objects.get( code=code, class_number=class_number, semester=Semester_object, name=subject_name) ## Update it for which is not specified (code, class_number, semester and name) Subject_object.category = Category_object Subject_object.department = Department_objects[cur_dept] Subject_object.plan = plan_a if prof_name != "": Subject_object.professor = prof_name if capacity != 0: Subject_object.capacity = capacity if credit != "": Subject_object.credit = credit Subject_object.save() try: print("Subject " + subject_name + "(" + code + ") is updated") except: print("Updated!") except ObjectDoesNotExist: ## There is no matching subject. do once more Subject_object = Subject( name=subject_name, code=code, category=Category_object, department=Department_objects[cur_dept], plan=plan_a, professor=prof_name, class_number=class_number, capacity=capacity, credit=credit, semester=Semester_object) Subject_object.save() try: print("Subject " + subject_name + "(" + code + ") is created") except: print("Created!") ## Try to find out Period or make it ## Update period just in case Period_objects = Period.objects.filter(subject=Subject_object) ## update for existing time if time_place_num - len(Period_objects) >= 0: for period_object_number in range(len(Period_objects)): Period_object = Period_objects[period_object_number] Period_object.place = place[period_object_number] Period_object.start = start_time[period_object_number] Period_object.end = end_time[period_object_number] Period_object.mon = '월' in days[period_object_number] Period_object.tue = '화' in days[period_object_number] Period_object.wed = '수' in days[period_object_number] Period_object.thu = '목' in days[period_object_number] Period_object.fri = '금' in days[period_object_number] Period_object.save() ## Create Period objects for not yet created time for number in range(time_place_num - len(Period_objects)): new_object_number = number + len(Period_objects) #print("Cur # " + str(new_object_number) + " with " + start_hour[new_object_number] + ":" + start_min[new_object_number]) Period_object = Period( subject=Subject_object, place=place[new_object_number], start=start_time[new_object_number], end=end_time[new_object_number], mon='월' in days[new_object_number], tue='화' in days[new_object_number], wed='수' in days[new_object_number], thu='목' in days[new_object_number], fri='금' in days[new_object_number]) Period_object.save() if limit: sleep(0.5) cur_page += 1 cur_page = 1
def get_courses(self): all_courses = Subject.query().fetch() return self.render_json({"success": True, "courses": [course.to_dict() for course in all_courses]})