def insertSuccess(self, msg):
"""
success crawle the article msg, insert into the successed db, insert into mongodb
"""
try:
self.mysql_client.begin()
# print article
# print msg["url"]
article = self.mysql_client.getOne("select * from failed_url where url=%s", (msg["url"], ))
if article != False:
article = self.mysql_client.delete("delete from failed_url where url=%s", (msg["url"], ))
LOGGER.info("delete the article from failed_url: %s", msg["url"])
article = self.mysql_client.getOne("select * from successed_url where url=%s", (msg["url"], ))
if article != False:
LOGGER.info("repeat crawler the article give up save: %s", msg["url"])
return
self.mongo_client.save(msg)
LOGGER.debug("insert into mongo: %s@%s" %(msg["title"], msg["url"]))
self.mysql_client.insertOne("insert into successed_url(url, tag, sub_tag, version, create_time) values(%s, %s, %s, %s, %s)", \
(msg["url"], msg["tag"], msg["sub_tag"], VERSION, msg["create_time"]));
LOGGER.debug("insert successed_url %s" %(msg["url"], ))
self.mysql_client.end("commit")
except Exception, e:
self.mysql_client.end("rollback")
self.mysql_client.begin()
self.insertFailed(msg)
LOGGER.error("insert into mongo/successed_url error: %s" %(msg["url"]))
LOGGER.error(traceback.format_exc())
def _process(self, msg):
'''
This is the main message process method, normally we have all the process
logic here
'''
try:
url, title, tag, sub_tag, abstract = self.pharseMsg(msg)
crawler = self.crawlerMapper[tag][sub_tag]
crawler.crawlArticle(msg)
except Exception, e:
LOGGER.error(traceback.format_exc())
LOGGER.error(e)
def _process(self, msg):
'''
This is the main message process method, normally we have all the process
logic here
'''
try:
url, title, tag, sub_tag, abstract = self.pharseMsg(msg)
crawler = self.crawlerMapper[tag][sub_tag]
crawler.crawlArticle(msg)
except Exception, e:
LOGGER.error(traceback.format_exc())
LOGGER.error(e)
def insertSuccess(self, msg):
"""
success crawle the article msg, insert into the successed db, insert into mongodb
"""
try:
self.mysql_client.begin()
# print article
# print msg["url"]
article = self.mysql_client.getOne(
"select * from failed_url where url=%s", (msg["url"], ))
if article != False:
article = self.mysql_client.delete(
"delete from failed_url where url=%s", (msg["url"], ))
LOGGER.info("delete the article from failed_url: %s",
msg["url"])
article = self.mysql_client.getOne(
"select * from successed_url where url=%s", (msg["url"], ))
if article != False:
LOGGER.info("repeat crawler the article give up save: %s",
msg["url"])
return
self.mongo_client.save(msg)
LOGGER.debug("insert into mongo: %s@%s" %
(msg["title"], msg["url"]))
self.mysql_client.insertOne("insert into successed_url(url, tag, sub_tag, version, create_time) values(%s, %s, %s, %s, %s)", \
(msg["url"], msg["tag"], msg["sub_tag"], VERSION, msg["create_time"]))
LOGGER.debug("insert successed_url %s" % (msg["url"], ))
self.mysql_client.end("commit")
except Exception, e:
self.mysql_client.end("rollback")
self.mysql_client.begin()
self.insertFailed(msg)
LOGGER.error("insert into mongo/successed_url error: %s" %
(msg["url"]))
LOGGER.error(traceback.format_exc())
def insertFailed(self, msg):
"""
insert into failed_url
"""
try:
self.mysql_client.begin()
article = self.mysql_client.getOne("select * from failed_url where url=%s", (msg["url"], ))
if article == False:
self.mysql_client.insertOne("insert into failed_url(url, tag, sub_tag, version, create_time) values(%s, %s, %s, %s, %s)", \
(msg["url"], msg["tag"], msg["sub_tag"], VERSION, msg["create_time"]));
LOGGER.debug("insert failed_url %s" %(msg["url"], ))
else:
self.mysql_client.update("update failed_url set count = count+1 where url = %s", (msg["url"], ))
LOGGER.debug("update failed_url %s" %(msg["url"], ))
self.mysql_client.end("commit")
LOGGER.debug("commit failed_url %s" %(msg["url"], ))
except Exception, e:
LOGGER.error(traceback.format_exc())
self.mysql_client.end("rollback")
def crawlArticle(self, msg):
"""
crawler the article referer by msg
"""
url = msg["url"]
try:
driver = webdriver.PhantomJS(PHANTOMJS_PATH)
driver.set_page_load_timeout(10)
article = self.mysql_client.getOne(
"select * from successed_url where url=%s", (msg["url"], ))
if article != False:
LOGGER.info("repeat crawler the article give up save: %s",
msg["url"])
return
LOGGER.debug("start extractor from %s" % (url, ))
driver.get(url)
try:
#scroll bar set from bottom to top, make the page load all
js = "var q=document.documentElement.scrollTop=10000"
driver.execute_script(js)
js = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js)
articles = self.pharseContext(driver)
msg["text"] = articles
self.insertSuccess(msg)
except Exception, e:
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" % (msg["url"], ))
self.insertFailed(msg)
except TimeoutException, e:
#scroll bar set from bottom to top, make the page load all
try:
js = "var q=document.documentElement.scrollTop=10000"
driver.execute_script(js)
js = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js)
# title = driver.find_element_by_css_selector("h1[id=\"h1title\"]").text
articles = self.pharseContext(driver)
msg["text"] = articles
self.insertSuccess(msg)
except Exception, e:
self.insertFailed(msg)
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" % (msg["url"], ))
driver.quit()
def run(self):
try:
LOGGING = {'version': 1 }
QUEUE_NAME = "news_article"
LOGGER.info("start the news crawler")
threadCount = CRAWLER_THREAD_COUNT
messageHandlerList = []
workThreadList = []
for _ in range(threadCount):
messageHandler = CrawlerMessageHandler(crawlerMapper)
messageHandler.set_inputmessage(QUEUE_NAME)
messageHandlerList.append(messageHandler)
workerThread = threading.Thread(target=messageHandler.start,args=(LOGGING))
workerThread.start()
workThreadList.append(workerThread)
for worker in workThreadList:
worker.join()
except Exception,e:
LOGGER.error(traceback.format_exc())
def crawlArticle(self, msg):
"""
crawler the article referer by msg
"""
url = msg["url"]
try:
driver = webdriver.PhantomJS(PHANTOMJS_PATH)
driver.set_page_load_timeout(10)
article = self.mysql_client.getOne("select * from successed_url where url=%s", (msg["url"], ))
if article != False:
LOGGER.info("repeat crawler the article give up save: %s", msg["url"])
return
LOGGER.debug("start extractor from %s" %(url, ))
driver.get(url)
try:
#scroll bar set from bottom to top, make the page load all
js = "var q=document.documentElement.scrollTop=10000"
driver.execute_script(js)
js = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js)
articles = self.pharseContext(driver)
msg["text"] = articles
self.insertSuccess(msg)
except Exception, e:
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" %(msg["url"],))
self.insertFailed(msg)
except TimeoutException, e:
#scroll bar set from bottom to top, make the page load all
try:
js = "var q=document.documentElement.scrollTop=10000"
driver.execute_script(js)
js = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js)
# title = driver.find_element_by_css_selector("h1[id=\"h1title\"]").text
articles = self.pharseContext(driver)
msg["text"] = articles
self.insertSuccess(msg)
except Exception, e:
self.insertFailed(msg)
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" %(msg["url"], ))
driver.quit()
def run(self):
try:
LOGGING = {'version': 1}
QUEUE_NAME = "news_article"
LOGGER.info("start the news crawler")
threadCount = CRAWLER_THREAD_COUNT
messageHandlerList = []
workThreadList = []
for _ in range(threadCount):
messageHandler = CrawlerMessageHandler(crawlerMapper)
messageHandler.set_inputmessage(QUEUE_NAME)
messageHandlerList.append(messageHandler)
workerThread = threading.Thread(target=messageHandler.start,
args=(LOGGING))
workerThread.start()
workThreadList.append(workerThread)
for worker in workThreadList:
worker.join()
except Exception, e:
LOGGER.error(traceback.format_exc())
def insertFailed(self, msg):
"""
insert into failed_url
"""
try:
self.mysql_client.begin()
article = self.mysql_client.getOne(
"select * from failed_url where url=%s", (msg["url"], ))
if article == False:
self.mysql_client.insertOne("insert into failed_url(url, tag, sub_tag, version, create_time) values(%s, %s, %s, %s, %s)", \
(msg["url"], msg["tag"], msg["sub_tag"], VERSION, msg["create_time"]))
LOGGER.debug("insert failed_url %s" % (msg["url"], ))
else:
self.mysql_client.update(
"update failed_url set count = count+1 where url = %s",
(msg["url"], ))
LOGGER.debug("update failed_url %s" % (msg["url"], ))
self.mysql_client.end("commit")
LOGGER.debug("commit failed_url %s" % (msg["url"], ))
except Exception, e:
LOGGER.error(traceback.format_exc())
self.mysql_client.end("rollback")
js = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js)
# title = driver.find_element_by_css_selector("h1[id=\"h1title\"]").text
articles = self.pharseContext(driver)
msg["text"] = articles
self.insertSuccess(msg)
except Exception, e:
self.insertFailed(msg)
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" %(msg["url"], ))
driver.quit()
except Exception, e:
self.insertFailed(msg)
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" %(msg["url"], ))
finally:
try:
driver.quit()
except Exception, e:
pass
def pharseTitle(self, driver):
return ""
def pharseAbstract(self, driver):
return ""
def pharseContext(self, driver):
"""
js = "var q=document.documentElement.scrollTop=0"
driver.execute_script(js)
# title = driver.find_element_by_css_selector("h1[id=\"h1title\"]").text
articles = self.pharseContext(driver)
msg["text"] = articles
self.insertSuccess(msg)
except Exception, e:
self.insertFailed(msg)
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" % (msg["url"], ))
driver.quit()
except Exception, e:
self.insertFailed(msg)
LOGGER.error(traceback.format_exc())
LOGGER.error("url: %s" % (msg["url"], ))
finally:
try:
driver.quit()
except Exception, e:
pass
def pharseTitle(self, driver):
return ""
def pharseAbstract(self, driver):
return ""
def pharseContext(self, driver):
"""
