示例#1
0
def start():
    rep = urllib2.Request(base_url, headers=header)
    response = urllib2.urlopen(rep)

    soup = BeautifulSoup(response.read(), "html.parser")

    history_list = []

    items = soup.select('.list li a')

    for item in items:
        history_dict = {}
        history_dict['title'] = item.get('title')
        history_dict['url'] = item.get('href')
        history_dict['content'] = startContent(history_dict['url'])
        # history.content = ''.join(startContent(history.url))
        # history.content = json.dumps(startContent(history.url), ensure_ascii=False)
        history_list.append(history_dict)

    print('读取完成')
    print('-----------------')


    DatabaseManager.insertHistoryList(history_list)
    # print type(history_list)
    # print json.dumps(history_list,ensure_ascii=False)
    return json.dumps(history_list,ensure_ascii=False)