def __init__(self, host, port):
SocketServer.ThreadingTCPServer.__init__(self, (host, port),
harvestManUrlHandler)
self.host, self.port = self.socket.getsockname()
# For storing data from crawlers
self.urls = PriorityQueue(0)
# For storing data from fetchers
self.urls2 = PriorityQueue(0)
self.urlmap = {}
self.flag = True
def __init__(self, host, port, protocol='tcp'):
self.urls = PriorityQueue(0)
self.port = port
self.host = host
self.protocol = protocol
self.urlmap = {}
asyncore.dispatcher_with_send.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.bind((self.host, port))
except socket.error:
raise
self.listen(5)
def __init__(self, host, port, protocol='tcp'):
# For storing data from crawlers
self.urls = PriorityQueue(0)
# For storing data from fetchers
self.urls2 = PriorityQueue(0)
self.port = port
self.host = host
self.protocol = protocol
self.urlmap = {}
# Count of gets
self.count1 = 0
self.count2 = 0
asyncore.dispatcher_with_send.__init__(self)
self.create_socket(socket.AF_INET, socket.SOCK_STREAM)
try:
self.bind((self.host, port))
self.port = self.getsockname()[1]
except socket.error:
raise
self.listen(20)
