def __init__(self):
     self.linker = Linker()
     self.err_handler = hanerr.ErrorHandler()
class BBCSpider:
    q = Queue()
    cnx = mysql.connector.connect(user='******', password='******'.format(password), database='crawl')
    cursor = cnx.cursor()

    def __init__(self):
        self.linker = Linker()
        self.err_handler = hanerr.ErrorHandler()

    def catch_news_urls(self):
        # 将BBC中的搜索结果的每一条新闻的 链接和对应的新闻名 都写入数据库
        i = 0
        flag = True
        while flag:
            # 自动翻页直至页面内容为空(只有空符号)
            i += 1
            page_url = 'http://www.bbc.co.uk/search/more?page={}&q=china%20vietnam'.format(i)
            print 'page:' + str(i)
            for try_times in range(5):
                try:
                    response = self.linker.direct(page_url, change=0)
                    response.raise_for_status()
                except:
                    continue
                if response.text.strip():
                    selector = etree.HTML(response.text)
                    urls = selector.xpath('//ol[@class="search-results results"]//a[@class="rs_touch"]/@href')
                    titles = selector.xpath('//ol[@class="search-results results"]/li//div/h1//text()')
                    BBCSpider.q.put((urls, titles))
                    break
                else:
                    print '[END]'
                    flag = False
                    break

    def catch_html(self, news_id, html, url):
        # 返回关于url的源代码
        if html:
            print '[自动跳过]'
            return
        for i in range(5):
            try:
                r = self.linker.direct(url)
                html = r.text
                BBCSpider.q.put((news_id, html))
                break
            except BaseException as err:
                self.err_handler.handle_linkerror(news_id, err)

    @staticmethod
    def catch_news_message(news_list):
        # 匹配新闻内容(发布时间、新闻简介、新闻主体)
        # 判断是否已经爬取

        # each[0]为id, each[1]为html, each[2]为match_time
        for each in news_list:
            if each[2]:
                print '[自动跳过]'
                continue
            try:
                # 调用MatchManager匹配内容
                body, public_time = match_message.MatchManager(str(each[1])).main_match()
                match_time = datetime.now().date()
                BBCSpider.q.put((each[0], public_time, body, match_time))  # (id, 发布时间, 新闻内容, 匹配时间)
            except BaseException as err:
                hanerr.ErrorHandler.err_q.put((each[0], err))  # (id, 错误)

    @staticmethod
    def threading_caturl():
        threads = [threading.Thread(target=BBCSpider().catch_news_urls), threading.Thread(target=insert_bbc_url)]
        for each in threads:
            each.start()
        for each in threads:
            each.join()

    @staticmethod
    def threading_download():
        query_sql = 'SELECT ID, html, url FROM bbc_test'
        BBCSpider.cursor.execute(query_sql)
        temp = BBCSpider.cursor.fetchall()
        threads = []
        for each in temp:
            threading.Thread(target=BBCSpider().catch_html, args=(each[0], each[1], each[2])).start()
        threading.Thread(target=update_bbc_html).start()
        threading.Thread(target=hanerr.basic_handle).start()

    @staticmethod
    def threading_match():
        query_sql = 'SELECT ID, html, match_time FROM bbc_test'
        BBCSpider.cursor.execute(query_sql)
        news_list = BBCSpider.cursor.fetchall()
        threads = [threading.Thread(target=BBCSpider.catch_news_message, args=(news_list,)),
                   threading.Thread(target=update_bbc_message),
                   threading.Thread(target=hanerr.basic_handle)]
        for each in threads:
            each.start()
        for each in threads:
            each.join()

    @staticmethod
    def has_url(url):
        query = 'SELECT 1 FROM bbc_test WHERE url="{}" limit 1'.format(url)
        BBCSpider.cursor.execute(query)
        if BBCSpider.cursor.fetchone():
            return True
        return False