Esempio n. 1
0
def fetch_database_urls():
    db = MyDB()
    query = "SELECT link, id FROM cs_sites"
    db.query(query)
    urls = {}
    for item in list(db.fetchall()):
        urls[item[1]] = item[0]
    db.commit()
    db.close()
    return urls
Esempio n. 2
0
        problem_info = {
            'problemId': items['id'],
            'title': items['title'],
            'type': items['type'],
            'content': items['content'],
            'points': items['score']
        }
        yield problem_info


if __name__ == '__main__':
    db = MyDB()
    table = 'problems'
    sql = """CREATE TABLE IF NOT EXISTS problems (
    problemId VARCHAR(255) NOT NULL,
    title VARCHAR(255) NOT NULL,
    type VARCHAR(64) NOT NULL,
    content TEXT,
    points INT NOT NULL,
    score INT,
    PRIMARY KEY (problemId))ENGINE=InnoDB DEFAULT CHARSET=utf8'
    """
    db.create_table(sql)
    get_id_list()
    for id in id_list:
        json = get_page(id)
        problem_info_list = parse_page(json)
        for problem_data in problem_info_list:
            db.savedata(problem_data, table)
    db.close()