def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
self.url_queue = [
'http://riweb.tibeica.com/crawl/'
] # 'http://fanacmilan.com/' 'http://riweb.tibeica.com/crawl/' 'http://www.greenworldmoldova.com/'
self.visited = {}
self.UTILS = Utils()
self.limit = 200
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
# WorkQueue is a convenient class that run slaves on a tasks queue
self.work_queue = WorkQueue(self.master)
def __init__(self, slaves):
# when creating the Master we tell it what slaves it can handle
self.master = Master(slaves)
