__author__ = 'PatrickYeh'
import receiver
import pickle
from pttcrawler import Logger
from kafka import KafkaProducer
log = Logger.getLogger("kafka_producer")
class kafka_producer(receiver.receiver):
def __init__(self):
self.topic = "Message"
def set_kafka_client(self,host,port):
self.producer = KafkaProducer(bootstrap_servers="{host}:{port}".format(host=host,port=port))
def set_topic(self,topic):
self.topic = topic
def send(self,obj_data):
log.debug("Broadcast Data")
self.producer.send(self.topic,obj_data)
class article_producer(kafka_producer):
def __init__(self):
self.set_topic("ptt_article")
class reply_producer(kafka_producer):
def __init__(self):
self.set_topic("ptt_reply")
# coding=utf-8
__author__ = 'PatrickYeh'
import re
from pttcrawler import Logger
from pttcrawler.WebRetriever import WebRetriever
from pttcrawler.Page import Page
from pttcrawler.Article import Article
from BeautifulSoup import BeautifulSoup
log = Logger.getLogger("PttBoard")
PAGE_REG = ".*index(?P<page_num>\d*).html"
ARTICLE_URL_REG = ".*/(?P<article_id>.*).html"
article_url_pattern = re.compile(ARTICLE_URL_REG)
class Board(Page):
def __init__(self,board_id="Gossiping"):
self.base_url = 'https://www.ptt.cc/bbs/{board_id}/index{page_idx}.html'
self.board_id = board_id
self.refresh()
def refresh(self):
self.html_raw_soup = self._fetch_data(self.url)
def _article_list_iter(self,lst_article_idx):
for article_idx in lst_article_idx:
yield Article(board_id=self.board_id,article_id=article_idx)
def get_articles(self,lst_article_list):
return self._article_list_iter(lst_article_list)
__author__ = 'PatrickYeh'
import threading,time,pickle,json
from pttcrawler import Logger
from pttcrawler.Board import Board
from pttcrawler.Article import Article
import kafka_producer
log = Logger.getLogger("Monitor")
class monitor(threading.Thread):
def __init__(self):
self.dict_receiver = {}
def set_receiver(self,name,obj_receiver):
self.dict_receiver[name] = obj_receiver
def get_receiver(self,name):
return self.dict_receiver[name]
def send(self,name,obj_data):
self.dict_receiver[name].send(obj_data)
def broadcast(self,obj_data):
for key in self.dict_receiver.keys():
self.dict_receiver[key].send(obj_data)
class article_monitor(monitor):
def __init__(self,board_id,article_id):
threading.Thread.__init__(self)
monitor.__init__(self)
# coding=utf-8
__author__ = 'Vetom'
import requests,time
from BeautifulSoup import BeautifulSoup
from pttcrawler import Logger
requests.packages.urllib3.disable_warnings()
log = Logger.getLogger("WebRetriever")
class WebRetriever():
def make_request(self,str_url):
log.debug("Make Query: {url}".format(url=str_url))
bool_pass = False
while not bool_pass:
try:
html_raw = requests.get(str_url,verify=False,cookies={'over18':'1'})
html_raw.encoding = 'utf-8'
bool_pass = True
except Exception as e:
log.error("URL:{url} - ERROR: {err}".format(url=str_url,err=e))
time.sleep(2)
return BeautifulSoup(html_raw.text)
if __name__ == '__main__':
data = WebRetriever().make_request('https://www.ptt.cc/bbs/joke/index.html')
print data
