def config_majors(self,majors_path):
""" Process Majors """
majors = dict()
for major,flag,course in self.file_reading_gen(majors_path,3,"\t",True):
if major not in majors:
major_instance = Major(major)
majors[major] = major_instance
try:
major_instance.process_course(flag,course)
except ValueError as v_err:
print(v_err)
return majors
def get_scope():
url = 'http://yz.chsi.com.cn'
try:
con = MySQLdb.connect(host='localhost',
passwd='123456',
db='kaoyan',
charset='utf8',
user='******')
cursor = con.cursor()
cursor.execute('select startid from tableid where id =1')
startid = cursor.fetchone()[0]
print("start id is %s" % startid)
for item in Major.select(Major.q.id >= startid):
aa = ""
print "item id is ", item.id
if not item.fanwei:
res = requests.get(url + item.url, timeout=50)
soup = BeautifulSoup(res.text, 'html.parser')
res2 = soup.select('#result_list table tbody tr')
for item2 in res2:
bb = ""
for item3 in item2.select('td')[1:]:
bb = bb + item3.text + "|"
aa = aa + bb + "|"
print aa
item.set(fanwei=aa)
cursor.execute('update tableid set startid = %d where id =1 ' %
item.id)
con.commit()
except requests.exceptions.Timeout as e:
# print str(e)
get_scope()
finally:
cursor.close()
con.close()
def add_student(self):
window = Toplevel()
window.wm_title("Student")
l1 = Label(window, text="ID:")
l1.grid(row=0, column=0)
self.id_text = StringVar()
e1 = Entry(window, textvariable=self.id_text)
e1.grid(row=0, column=1)
l2 = Label(window, text="Name:")
l2.grid(row=1, column=0)
self.id_name = StringVar()
e2 = Entry(window, textvariable=self.id_name)
e2.grid(row=1, column=1)
l3 = Label(window, text="Major:")
l3.grid(row=2, column=0)
self.variable = StringVar()
major = []
if len(Major.list_major()) > 0:
self.variable.set(Major.list_major()[0]) # default value
# create a list of string to be used in the method below
for i in Major.list_major():
major.append(str(*i))
w = OptionMenu(window, self.variable, *major)
print("HEREEEEEEEEEEEE")
print(self.variable.get())
w.grid(row=2, column=1)
else:
self.variable.set("")
w = OptionMenu(window, self.variable, "")
w.grid(row=2, column=1)
b1 = Button(window, text="insert data", command=self.insert_student)
b1.grid(row=3, column=0)
window.geometry("500x200")
window.mainloop()
def major_basic_populate():
Base.metadata.create_all(engine)
with open('../scrapers/majors.json', 'r') as f:
major_data = json.load(f)
for major in major_data:
session = Session()
major_db = Major(major["major_id"], major["name"])
major_db.add_major_data(
major["wage_growth_rate"], major["image_link"], major["is_stem"],
cast_int_wage(major["average_wage"]),
major["total_degrees_awarded_in_2015"],
cast_int_workforce(major["total_people_in_work_foce"]),
major["average_age_work_force"])
print("Added " + major["name"])
session.add(major_db)
session.commit()
session.close()
def majors_DB(self):
""" Structure: Department, Flag, Course
Method:
If major is already in the dict, then ignore, just update info.
As each time declare an instance, the required and electives would be an emplty set,
this would avoid clean the set by mistake.
"""
for major, flag, course in self.file_reader(self.majors_path):
if major not in self.majors:
self.majors[major] = Major(major)
self.majors[major].update_major(flag, course)
def parse_program (site):
program = Program ("")
parser = etree.HTMLParser(remove_blank_text=True)
tree = etree.parse (site, parser)
content = '/descendant::div[attribute::class="content-main"]'
place = tree.xpath (content)[0]
result = etree.tostring (place, pretty_print=True, method="html")
majors_path = content + "/descendant::table[contains(@class, 'majorTable')]"
for m in tree.xpath (majors_path):
m_path = tree.getpath(m)
title_path = m_path + "/thead/descendant::a[attribute::class='title']/text()"
major_title = tree.xpath (title_path)[0]
maj = Major (major_title)
course_path = m_path + "/descendant-or-self::table[contains(@class, 'courseTable')]"
for course in tree.xpath (course_path):
maj.courses = parse_course_table (program, tree, course)
program.majors.append (maj)
return program
def get_content():
for item in PreMajor.select(PreMajor.q.id >= 785):
print item.id
reqdata['ssdm'] = item.cityCode
reqdata['dwmc'] = item.schoolName
for pageno in xrange(1, item.pageNum + 1):
reqdata['pageno'] = pageno
# print reqdata
res2 = requests.post(queryurl, data=reqdata, timeout=50)
soup = BeautifulSoup(res2.text, 'lxml')
res3 = soup.select('#sch_list table tbody tr')
for item3 in res3:
college = item3.contents[1].text
major = item3.contents[3].text
direction = item3.contents[5].text
people = item3.contents[9].text
peoplenum = parrten.findall(people)[0]
newurl = item3.contents[11].a['href']
u = Major.select(Major.q.url == newurl)
if not u.count():
o = Major(city=item.city,
cityCode=item.cityCode,
schoolName=item.schoolName,
schoolCode=item.schoolCode,
college=college[5:],
collegeCode=college[1:4],
major=major[8:],
majorCode=major[1:7],
direction=direction[4:],
directionCode=direction[1:3],
peopleNum=peoplenum,
fanwei="",
url=newurl)
if o:
print("ok")
else:
print("failed")
print("it's over")
def findMajor(target, majors):
# Check if there is no target.
if not target:
return []
options = []
words = target.lower().split()
# Loop through allInstitution to find potential institution.
for major in majors:
label = majors[major]['Major'].lower()
if all([word in label for word in words]):
options.append(Major(majors[major]['Major'].title()))
return options
def findMajor(target, majors):
"""
Find the option that is exactly the same as users' targets or the options that contain users' targets.
"""
# Check if there is no target.
if not target:
return []
options = []
words = target.lower().split()
# Loop through allInstitution to find potential institution.
for major in majors:
label = major['label'].lower()
if all([word in label for word in words]):
options.append(Major(major['label']))
return options
def list_major(self):
window = Toplevel()
window.wm_title("List all Majors")
l1 = Label(window, text="Majors list:")
l1.grid(row=0, column=0, columnspan=6)
list1 = Listbox(window, height=6, width=35)
list1.grid(row=1, column=0, rowspan=6, columnspan=2)
sb1 = Scrollbar(window)
sb1.grid(row=2, column=2, rowspan=6)
list1.configure(yscrollcommand=sb1.set)
sb1.configure(command=list1.yview)
list1.delete(END, 0)
for rows in Major.list_major():
print("Major items" + str(rows))
list1.insert(END, rows)
window.geometry("500x200")
window.mainloop()
def insert_major(self):
print(self.id_major.get(), self.name_major.get())
major = Major(self.id_major.get(), self.name_major.get())
major.insert_major()