Пример #1
0
    def __init__(self):
        self.ip = None
        self.leader = None
        self.node = None
        self.is_leader = lambda :(self.leader == self.node)
        
        self.zk = zookeeper()

        self.zk.start()

        self.ensure_path()
        self.set_watch()
Пример #2
0
    def __init__(self, name='spider', workdir=None, store=None):
        info('__init__', 'start to init the spider..')
        Thread.__init__(self)

        if store == None and workdir == None:
            raise Exception('set store value or workdir to create spider')
        self.store = store if store != None else FileStore(workdir, name)

        self.name = name
        self.start_time = time.time()
        self.pause_seconds = 1
        self.urls = []
        self.zk = zookeeper()
        self.zk.start()
        self.zk.ensure_path("/spider/data/running/%s" % self.website())
        self.zk.ensure_path("/spider/spiders/%s" % self.website())
        #self.register()
        info('__init__', 'init ok.')
Пример #3
0
 def __init__(self,
              name='spider',
              workdir=None,
              store=None):
     info('__init__', 'start to init the spider..')
     Thread.__init__(self)
     
     if store == None and workdir == None:
         raise Exception('set store value or workdir to create spider')
     self.store = store if store != None else  FileStore(workdir, name)
     
     self.name = name
     self.start_time = time.time()
     self.pause_seconds = 1
     self.urls = []
     self.zk = zookeeper()
     self.zk.start()
     self.zk.ensure_path("/spider/data/running/%s" % self.website())
     self.zk.ensure_path("/spider/spiders/%s" % self.website())
     #self.register()
     info('__init__', 'init ok.')