コード例 #1
0
    def __init__(self,
                 job,
                 data_dir,
                 master=None,
                 local=None,
                 nodes=None,
                 context=None,
                 logger=None,
                 copies=1,
                 force=False):
        BasicWorkerJobLoader.__init__(self,
                                      job,
                                      data_dir,
                                      context=context,
                                      logger=logger,
                                      local=local,
                                      nodes=nodes,
                                      copies=copies,
                                      force=force)
        LimitionJobLoader.__init__(self, self.job, context=context)

        log_level = logging.INFO if not job.debug else logging.DEBUG
        if self.logger is None:
            self.logger = get_logger(
                name='cola_worker_%s' % self.job.real_name,
                filename=os.path.join(self.root, 'job.log'),
                basic_level=log_level)

        self.init_rate_clear()
コード例 #2
0
ファイル: loader.py プロジェクト: friedvan/cola
 def finish(self):
     all_pages = self.pages()
     
     self.release_lock(self.ready_lock)
     self.release_lock(self.finish_lock)
     
     LimitionJobLoader.finish(self)
     JobLoader.finish(self)
     self.stop_logger_server()
     
     try:
         for handler in self.logger.handlers:
             handler.close()
     except:
         pass
         
     if self.client is not None:
         rpc_client = '%s:%s' % (
             self.client.split(':')[0], 
             main_conf.client.port
         )
         client_call(rpc_client, 'stop', ignore=True)
         
     self.logger.info('All nodes finishes visiting pages size: %s' % all_pages)
     self.stopped = True
コード例 #3
0
ファイル: loader.py プロジェクト: MwzkQmuUZkFLbXm/cola
 def __init__(self, job, data_dir, nodes, client=None,
              context=None, copies=1, force=False):
     ctx = context or job.context
     master_port = ctx.job.master_port
     local = '%s:%s' % (get_ip(), master_port)
     
     JobLoader.__init__(self, job, data_dir, local, 
                        context=ctx, copies=copies, force=force)
     LimitionJobLoader.__init__(self, job, context=ctx)
     
     # check
     self.check()
     
     self.nodes = nodes
     self.not_registered = self.nodes[:]
     self.not_finished = self.nodes[:]
     
     # mq
     self.mq_client = MessageQueueClient(self.nodes, copies=copies)
     
     # lock
     self.ready_lock = threading.Lock()
     self.ready_lock.acquire()
     self.finish_lock = threading.Lock()
     self.finish_lock.acquire()
     
     # logger
     self.logger = get_logger(
         name='cola_master_%s'%self.job.real_name,
         filename=os.path.join(self.root, 'job.log'),
         is_master=True)
     self.client = client
     self.client_handler = None
     if self.client is not None:
         self.client_handler = add_log_client(self.logger, self.client)
     
     self.init_rpc_server()
     self.init_rate_clear()
     self.init_logger_server(self.logger)
     
     # register rpc server
     self.rpc_server.register_function(self.client_stop, 'client_stop')
     self.rpc_server.register_function(self.ready, 'ready')
     self.rpc_server.register_function(self.worker_finish, 'worker_finish')
     self.rpc_server.register_function(self.complete, 'complete')
     self.rpc_server.register_function(self.error, 'error')
     self.rpc_server.register_function(self.get_nodes, 'get_nodes')
     self.rpc_server.register_function(self.apply, 'apply')
     self.rpc_server.register_function(self.require, 'require')
     self.rpc_server.register_function(self.stop, 'stop')
     self.rpc_server.register_function(self.add_node, 'add_node')
     self.rpc_server.register_function(self.remove_node, 'remove_node')
     
     # register signal
     signal.signal(signal.SIGINT, self.signal_handler)
     signal.signal(signal.SIGTERM, self.signal_handler)
コード例 #4
0
ファイル: loader.py プロジェクト: bingyupj/cola
 def __init__(self, job, data_dir, master=None, local=None, nodes=None, 
              context=None, logger=None, copies=1, force=False):
     BasicWorkerJobLoader.__init__(self, job, data_dir, context=context, logger=logger,
                                   local=local, nodes=nodes, copies=copies, force=force)
     LimitionJobLoader.__init__(self, self.job, context=context)
     
     if self.logger is None:
         self.logger = get_logger(
             name='cola_worker_%s'%self.job.real_name,
             filename=os.path.join(self.root, 'job.log'))
         
     self.init_rate_clear()
コード例 #5
0
    def finish(self):
        self.release_lock(self.ready_lock)
        self.release_lock(self.finish_lock)

        LimitionJobLoader.finish(self)
        JobLoader.finish(self)
        self.stop_logger_server()

        try:
            for handler in self.logger.handlers:
                handler.close()
        except:
            pass

        if self.client is not None:
            rpc_client = '%s:%s' % (self.client.split(':')[0],
                                    main_conf.client.port)
            client_call(rpc_client, 'stop', ignore=True)

        self.stopped = True
コード例 #6
0
ファイル: loader.py プロジェクト: friedvan/cola
 def error(self, obj):
     LimitionJobLoader.error(self, obj)
     BasicWorkerJobLoader.error(self, obj)
コード例 #7
0
ファイル: loader.py プロジェクト: friedvan/cola
 def complete(self, obj):
     BasicWorkerJobLoader.complete(self, obj)
     return LimitionJobLoader.complete(self, obj)
コード例 #8
0
ファイル: loader.py プロジェクト: friedvan/cola
 def stop(self):
     LimitionJobLoader.stop(self)
     BasicWorkerJobLoader.stop(self)
コード例 #9
0
ファイル: loader.py プロジェクト: friedvan/cola
 def finish(self):
     LimitionJobLoader.finish(self)
     BasicWorkerJobLoader.finish(self)
コード例 #10
0
 def error(self, obj):
     LimitionJobLoader.error(self, obj)
     BasicWorkerJobLoader.error(self, obj)
コード例 #11
0
 def complete(self, obj):
     BasicWorkerJobLoader.complete(self, obj)
     return LimitionJobLoader.complete(self, obj)
コード例 #12
0
 def stop(self):
     LimitionJobLoader.stop(self)
     BasicWorkerJobLoader.stop(self)
コード例 #13
0
 def finish(self):
     LimitionJobLoader.finish(self)
     BasicWorkerJobLoader.finish(self)
コード例 #14
0
    def __init__(self,
                 job,
                 data_dir,
                 nodes,
                 local_ip=None,
                 client=None,
                 context=None,
                 copies=1,
                 force=False):
        ctx = context or job.context
        master_port = ctx.job.master_port
        if local_ip is None:
            local_ip = get_ip()
        else:
            choices_ips = get_ips()
            if local_ip not in choices_ips:
                raise ValueError('IP address must be one of (%s)' %
                                 ','.join(choices_ips))
        local = '%s:%s' % (local_ip, master_port)

        JobLoader.__init__(self,
                           job,
                           data_dir,
                           local,
                           context=ctx,
                           copies=copies,
                           force=force)
        LimitionJobLoader.__init__(self, job, context=ctx)

        # check
        self.check()

        self.nodes = nodes
        self.not_registered = self.nodes[:]
        self.not_finished = self.nodes[:]

        # mq
        self.mq_client = MessageQueueClient(self.nodes, copies=copies)

        # lock
        self.ready_lock = threading.Lock()
        self.ready_lock.acquire()
        self.finish_lock = threading.Lock()
        self.finish_lock.acquire()

        # logger
        self.logger = get_logger(name='cola_master_%s' % self.job.real_name,
                                 filename=os.path.join(self.root, 'job.log'),
                                 is_master=True)
        self.client = client
        self.client_handler = None
        if self.client is not None:
            self.client_handler = add_log_client(self.logger, self.client)

        self.init_rpc_server()
        self.init_rate_clear()
        self.init_logger_server(self.logger)

        # register rpc server
        self.rpc_server.register_function(self.client_stop, 'client_stop')
        self.rpc_server.register_function(self.ready, 'ready')
        self.rpc_server.register_function(self.worker_finish, 'worker_finish')
        self.rpc_server.register_function(self.complete, 'complete')
        self.rpc_server.register_function(self.error, 'error')
        self.rpc_server.register_function(self.get_nodes, 'get_nodes')
        self.rpc_server.register_function(self.apply, 'apply')
        self.rpc_server.register_function(self.require, 'require')
        self.rpc_server.register_function(self.stop, 'stop')
        self.rpc_server.register_function(self.add_node, 'add_node')
        self.rpc_server.register_function(self.remove_node, 'remove_node')

        # register signal
        signal.signal(signal.SIGINT, self.signal_handler)
        signal.signal(signal.SIGTERM, self.signal_handler)