def __init__(self, job, nodes, rpc_server, context=None, copies=2): self.job = job self.ctx = context or job.context self.nodes = nodes self.mq_client = MessageQueueClient(self.nodes, copies=copies) self.not_registered = self.nodes[:] self.is_ready = False self.stopped = False # destination size self.size = self.ctx.job.size self.limit_size = self.size > 0 self.finishes = 0 # speed limits self.limits = self.ctx.job.limits self.limit_speed = self.limits > 0 self.in_minute = 0 # register rpc server rpc_server.register_function(self.ready, 'ready') rpc_server.register_function(self.complete, 'complete') rpc_server.register_function(self.get_nodes, 'get_nodes') rpc_server.register_function(self.require, 'require') rpc_server.register_function(self.stop, 'stop') rpc_server.register_function(self.add_node, 'add_node') rpc_server.register_function(self.remove_node, 'remove_node') # register signal signal.signal(signal.SIGINT, self.signal_handler) signal.signal(signal.SIGTERM, self.signal_handler)
def put_starts(master=None): if master is None: nodes = ['%s:%s' % (get_ip(), getattr(user_config.job, 'port'))] else: nodes = client_call(master, 'get_nodes') mq_client = MessageQueueClient(nodes) with open(keywords_f) as f: keys = [] size = 0 for keyword in f.xreadlines(): keys.append(keyword) size += 1 if size >= PUTSIZE: mq_client.put(keys) size = 0 keys = [] if len(keys) > 0: mq_client.put(keys)
def setUp(self): ports = (11111, 11211, 11311) self.nodes = ['localhost:%s' % port for port in ports] self.dirs = [tempfile.mkdtemp() for _ in range(2 * len(ports))] self.size = len(ports) for i in range(self.size): setattr(self, 'rpc_server%s' % i, ColaRPCServer(('localhost', ports[i]))) setattr( self, 'mq%s' % i, MessageQueue(self.nodes[:], self.nodes[i], getattr(self, 'rpc_server%s' % i))) getattr(self, 'mq%s' % i).init_store(self.dirs[2 * i], self.dirs[2 * i + 1]) thd = threading.Thread(target=getattr(self, 'rpc_server%s' % i).serve_forever) thd.setDaemon(True) thd.start() self.client = MessageQueueClient(self.nodes)
def put_starts(master=None): if master is None: master = ['%s:%s' % (get_ip(), getattr(user_config.master, 'port'))] print('master:%s' % master) jobs = client_call(master, 'runnable_jobs') app_name = '' for a, j in jobs.items(): if j == "douban movie": app_name = a break if not app_name: raise Exception('douban movie job has not upload') nodes = client_call(master, 'list_workers') addrs = [] default_addr = master.split(':')[0] for ap, s in nodes: a, p = ap.split(':') if a.lower() == 'localhost': addrs.append('%s:%s' % (default_addr, p)) else: addrs.append(ap) mq_client = MessageQueueClient(addrs, app_name) print('get:%s' % mq_client.get()) urls = [] size = 0 for url in starts: urls.append(url) size += 1 if size >= PUTSIZE: mq_client.put(urls) size = 0 urls = [] if len(urls) > 0: mq_client.put(urls)
def __init__(self, job, data_dir, nodes, local_ip=None, client=None, context=None, copies=1, force=False): ctx = context or job.context master_port = ctx.job.master_port if local_ip is None: local_ip = get_ip() else: choices_ips = get_ips() if local_ip not in choices_ips: raise ValueError('IP address must be one of (%s)' % ','.join(choices_ips)) local = '%s:%s' % (local_ip, master_port) JobLoader.__init__(self, job, data_dir, local, context=ctx, copies=copies, force=force) LimitionJobLoader.__init__(self, job, context=ctx) # check self.check() self.nodes = nodes self.not_registered = self.nodes[:] self.not_finished = self.nodes[:] # mq self.mq_client = MessageQueueClient(self.nodes, copies=copies) # lock self.ready_lock = threading.Lock() self.ready_lock.acquire() self.finish_lock = threading.Lock() self.finish_lock.acquire() # logger self.logger = get_logger(name='cola_master_%s' % self.job.real_name, filename=os.path.join(self.root, 'job.log'), is_master=True) self.client = client self.client_handler = None if self.client is not None: self.client_handler = add_log_client(self.logger, self.client) self.init_rpc_server() self.init_rate_clear() self.init_logger_server(self.logger) # register rpc server self.rpc_server.register_function(self.client_stop, 'client_stop') self.rpc_server.register_function(self.ready, 'ready') self.rpc_server.register_function(self.worker_finish, 'worker_finish') self.rpc_server.register_function(self.complete, 'complete') self.rpc_server.register_function(self.error, 'error') self.rpc_server.register_function(self.get_nodes, 'get_nodes') self.rpc_server.register_function(self.apply, 'apply') self.rpc_server.register_function(self.require, 'require') self.rpc_server.register_function(self.stop, 'stop') self.rpc_server.register_function(self.add_node, 'add_node') self.rpc_server.register_function(self.remove_node, 'remove_node') # register signal signal.signal(signal.SIGINT, self.signal_handler) signal.signal(signal.SIGTERM, self.signal_handler)