def main():
t1 = task("http://www.laurentluce.com/posts/python-threads-synchronization-locks-rlocks-semaphores-conditions-events-and-queues/")
t2 = task("http://stackoverflow.com/questions/15651128/in-this-semaphore-example-is-it-necessary-to-lock-for-refill-and-buy")
t3 = task("http://bbs.byr.cn/")
event = Event()
tasks = TaskQueue(event)
pages = TaskQueue(None)
tasks.add(t1)
tasks.add(t2)
tasks.add(t3)
taskLock = BoundedSemaphore(tasks.numOfNewTasks)
pageLock = BoundedSemaphore(1)
f = open("test.txt",'w')
Connector0 = Connector(tasks,taskLock,pages,pageLock,event,'',f, 3000)
Connector1 = Connector(tasks,taskLock,pages,pageLock,event,'',f, 3001)
Connector0.start()
Connector1.start()
Crawler0 = Crawler('',3000)
Crawler1 = Crawler('',3001)
Crawler0.start()
Crawler1.start()
Connector1.join()
Connector0.join()
Crawler0.join()
Crawler1.join()
f.close()
class Connector(Thread):
"""docstring for Connector"""
def __init__(self, taskQueue, TaskLock, PageQueue, PageLock, event, host, output, port):
Thread.__init__(self)
self.taskQueue = taskQueue
self.TaskLock = TaskLock
self.PageQueue = PageQueue
self.PageLock = PageLock
self.event = event
self.host = host
self.output = output
self.port = port
self.doneTask = TaskQueue(None)
self.donePage = TaskQueue(None)
self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)
# @param port is the wanted port
def prepare(self):
self.socket.bind((self.host,self.port))
print "Listening on the port ", self.port
self.socket.listen(5)
self.conn, self.addr = self.socket.accept()
print 'connected by', self.addr
def run(self):
self.prepare()
while 1:
self.event.wait()
with self.TaskLock:
task = self.taskQueue.get()
task.state = "working"
self.conn.send(task.url)
page = self.conn.recv(2048)
if task.type == "list":
with self.TaskLock:
for newTask in page:
self.taskQueue.add(newTask)
task.state = "done"
self.doneTask.add(task)
else:
with self.PageLock:
self.PageQueue.add(page)
self.output.write(page)
with self.TaskLock:
task.state = "done"
self.doneTask.add(task)
if self.taskQueue.numOfNewTasks == 0:
self.conn.send("")
break
self.conn.close()
return
def __init__(self, taskQueue, TaskLock, PageQueue, PageLock, event, host, output, port): Thread.__init__(self) self.taskQueue = taskQueue self.TaskLock = TaskLock self.PageQueue = PageQueue self.PageLock = PageLock self.event = event self.host = host self.output = output self.port = port self.doneTask = TaskQueue(None) self.donePage = TaskQueue(None) self.socket = socket.socket(socket.AF_INET,socket.SOCK_STREAM)