def __init__(self):
self.ip = None
self.leader = None
self.node = None
self.is_leader = lambda :(self.leader == self.node)
self.zk = zookeeper()
self.zk.start()
self.ensure_path()
self.set_watch()
def __init__(self, name='spider', workdir=None, store=None):
info('__init__', 'start to init the spider..')
Thread.__init__(self)
if store == None and workdir == None:
raise Exception('set store value or workdir to create spider')
self.store = store if store != None else FileStore(workdir, name)
self.name = name
self.start_time = time.time()
self.pause_seconds = 1
self.urls = []
self.zk = zookeeper()
self.zk.start()
self.zk.ensure_path("/spider/data/running/%s" % self.website())
self.zk.ensure_path("/spider/spiders/%s" % self.website())
#self.register()
info('__init__', 'init ok.')
def __init__(self,
name='spider',
workdir=None,
store=None):
info('__init__', 'start to init the spider..')
Thread.__init__(self)
if store == None and workdir == None:
raise Exception('set store value or workdir to create spider')
self.store = store if store != None else FileStore(workdir, name)
self.name = name
self.start_time = time.time()
self.pause_seconds = 1
self.urls = []
self.zk = zookeeper()
self.zk.start()
self.zk.ensure_path("/spider/data/running/%s" % self.website())
self.zk.ensure_path("/spider/spiders/%s" % self.website())
#self.register()
info('__init__', 'init ok.')