def fetch_database_urls():
db = MyDB()
query = "SELECT link, id FROM cs_sites"
db.query(query)
urls = {}
for item in list(db.fetchall()):
urls[item[1]] = item[0]
db.commit()
db.close()
return urls
problem_info = {
'problemId': items['id'],
'title': items['title'],
'type': items['type'],
'content': items['content'],
'points': items['score']
}
yield problem_info
if __name__ == '__main__':
db = MyDB()
table = 'problems'
sql = """CREATE TABLE IF NOT EXISTS problems (
problemId VARCHAR(255) NOT NULL,
title VARCHAR(255) NOT NULL,
type VARCHAR(64) NOT NULL,
content TEXT,
points INT NOT NULL,
score INT,
PRIMARY KEY (problemId))ENGINE=InnoDB DEFAULT CHARSET=utf8'
"""
db.create_table(sql)
get_id_list()
for id in id_list:
json = get_page(id)
problem_info_list = parse_page(json)
for problem_data in problem_info_list:
db.savedata(problem_data, table)
db.close()