def __init__(self, workqueue):
threading.Thread.__init__(self)
self.workQueue = workqueue
self.setDaemon(True)
self.start()
self.mongodb = MongoDB()
self.checkMongoDB = TempMongoDB()
def __init__(self, workQueue, saveQueue, timeout=30):
threading.Thread.__init__(self)
self.timeout = timeout
self.setDaemon(True)
self.workQueue = workQueue
self.saveQueue = saveQueue
self.mongodb = MongoDB()
self.start()
def __init__(self):
self.comment = NewsComment()
self.mongo = MongoDB()
self.conn = MySQLdb.connect('localhost',
'root',
'1995',
'newsurl',
charset='utf8',
use_unicode=True)
self.cursor = self.conn.cursor()
self.message = NewsMessage()
def __init__(self):
self.mongo = MongoDB()
def __init__(self):
self.comment = NewsComment()
self.mongo = MongoDB()
self.huan = huanCun()
def __init__(self):
self.comment = NewsComment()
self.mongo = MongoDB()
self.huan = huanCun()
'''self.genzong = genZong()'''
def __init__(self, savequeue):
self.saveQueue = savequeue
self.check_mongodb = TempMongoDB()
self.update_mongodb = MongoDB()
updateUrl_Queue = Queue.Queue()
# First: check old news' url for update
Updater = CheckUpdate(savequeue=updateUrl_Queue)
Updater.run()
CommentPool = ThreadPoolOfCommentThread(num_of_threads=10, workqueue=updateUrl_Queue)
CommentPool.wait_for_complete()
print "UPDATE: Complete checking for the old news!"
# Second: Get all news' url for json
for url in Urls:
StartUrlQueue.put(url)
fetch_1 = MyFetchThreadFirst(workQueue=StartUrlQueue, saveQueue=Json_Url_Queue)
fetch_1.join()
# Third: Parse json data and get news' html and parse to generate the message_dict then save it
print "FETCH: There are %d newest news has been fetched today" % Json_Url_Queue.qsize()
FetchSecondPool = ThreadPoolOfFetchSecond(num_of_threads=6, workqueue=Json_Url_Queue, savequeue=urlInfo_Queue)
FetchSecondPool.wait_for_complete()
# Forth: Request the comment_url and save comment's message_dict
CommentPool = ThreadPoolOfCommentThread(num_of_threads=10, workqueue=urlInfo_Queue)
CommentPool.wait_for_complete()
start_uptime, end_uptime = getCondition()
time_condition = {"do_time": {"$gte": start_uptime, "$lt": end_uptime},
"zhan_dian": "网易新闻"}
temp_mongo = MongoDB()
num_of_comments = temp_mongo.get_comment_data(time_condition)
print "FETCH: There are %d comments has been fetched today" % num_of_comments
print "TIME: Total spent %d seconds" % (time.time() - start_moment)
print "SLEEP: Mission complete, start to sleeping.... "
sleep_seconds = DeltaSeconds()
time.sleep(sleep_seconds)
def __init__(self):
self.comment = NewsComment()
self.mongo = MongoDB()
self.f = open('test.txt', "r+")
self.i = 0
self.url_list =[]
