def get_departments():
with codecs.open('departments/departments.txt', 'r', 'utf-8') as f:
departments = f.readlines()
departments = [line.strip().split('::') for line in departments]
return [
Department(department[0], department[1])
if len(department) > 1 else Department(department[0], None)
for department in departments
]
def updateDepartment(department_id):
if not request.json or not 'name' in request.json or not 'id' in request.json:
return jsonify({"result": False, "msg": "Failed to Update Department"})
department = Department(request.json['name'], request.json['id'])
result = Department.updateDepartment(department, mysql)
if result is True:
return jsonify({
"result": True,
"msg": "Successfully Updated Department!"
})
return jsonify({"result": False, "msg": "Failed to Update Department!"})
def addDepartment():
if not request.json or not 'name' in request.json:
return jsonify({"result": False, "msg": "Failed to Add Department"})
department = Department(request.json['name'])
result = Department.addDepartment(department, mysql)
if result is True:
return jsonify({
"result": True,
"msg": "Department Added Successfully!"
})
return jsonify({"result": False, "msg": "Failed to Add Department!"})
def scrapeDepartment():
print("*************** Scraping Departments *********************")
url = 'https://www.dal.ca/academics/faculties.html'
soup = get_soup(url)
main_div = soup.find_all("div", class_="expandingSubsection section")
d_list = list()
for child in main_div:
a_tags = child.find_all("a")
for a_tag in a_tags:
txt = a_tag.get_text().strip()
link = a_tag.get('href')
d_list.append(Department(txt, link))
pass
generate_id(d_list)
XmlList().from_list(d_list).save("department.xml")
def createDepartment(self, info):
departmentName = info['departmentName'].strip()
departmentID = self.generateID(departmentName)
now = datetime.now()
try:
(status, reason) = self.__dupCheck(info=info)
if status is not True:
return (False, ErrorInfo['TENDER_46'])
department = Department(departmentID=departmentID,
departmentName=departmentName,
createTime=now)
db.session.add(department)
db.session.commit()
return (True, departmentID)
except Exception as e:
print e
traceback.print_exc()
errorInfo = ErrorInfo['TENDER_02']
errorInfo['detail'] = str(e)
db.session.rollback()
return (False, errorInfo)
import requests
from bs4 import BeautifulSoup
from spider import build_url, analyse_department_page, get_department_topics, spider, get_url
from models.Department import Department
from tools.repo import get_departments
from tools.updates2html import updates2html
cfg = ConfigParser()
cfg.read(os.path.abspath('.') + '\config.ini')
print(cfg.sections())
smtp_server = cfg.get('email', 'smtp_server')
from_addr = cfg.get('email', 'from_addr')
password = cfg.get('email', 'password')
print(smtp_server)
print(from_addr)
print(password)
cookies = cfg.items('douban_cookie')
print(dict(cookies))
print(get_departments())
url = build_url('滨兴小区', 0)
print(BeautifulSoup(get_url(url).content, 'html.parser').get_text())
print(BeautifulSoup(get_url(url).content, 'html.parser').get_text())
print(analyse_department_page(requests.get(url).content))
print(get_department_topics(Department('白金海岸', '2018-06-05 19:18:53')))
res = spider(get_departments())
print(updates2html(res))
print(os.path.abspath('.') + '\config.ini')
