def __init__(self, job, nodes, rpc_server, context=None, copies=2):
self.job = job
self.ctx = context or job.context
self.nodes = nodes
self.mq_client = MessageQueueClient(self.nodes, copies=copies)
self.not_registered = self.nodes[:]
self.is_ready = False
self.stopped = False
# destination size
self.size = self.ctx.job.size
self.limit_size = self.size > 0
self.finishes = 0
# speed limits
self.limits = self.ctx.job.limits
self.limit_speed = self.limits > 0
self.in_minute = 0
# register rpc server
rpc_server.register_function(self.ready, 'ready')
rpc_server.register_function(self.complete, 'complete')
rpc_server.register_function(self.get_nodes, 'get_nodes')
rpc_server.register_function(self.require, 'require')
rpc_server.register_function(self.stop, 'stop')
rpc_server.register_function(self.add_node, 'add_node')
rpc_server.register_function(self.remove_node, 'remove_node')
# register signal
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
def put_starts(master=None):
if master is None:
nodes = ['%s:%s' % (get_ip(), getattr(user_config.job, 'port'))]
else:
nodes = client_call(master, 'get_nodes')
mq_client = MessageQueueClient(nodes)
with open(keywords_f) as f:
keys = []
size = 0
for keyword in f.xreadlines():
keys.append(keyword)
size += 1
if size >= PUTSIZE:
mq_client.put(keys)
size = 0
keys = []
if len(keys) > 0:
mq_client.put(keys)
def setUp(self):
ports = (11111, 11211, 11311)
self.nodes = ['localhost:%s' % port for port in ports]
self.dirs = [tempfile.mkdtemp() for _ in range(2 * len(ports))]
self.size = len(ports)
for i in range(self.size):
setattr(self, 'rpc_server%s' % i,
ColaRPCServer(('localhost', ports[i])))
setattr(
self, 'mq%s' % i,
MessageQueue(self.nodes[:], self.nodes[i],
getattr(self, 'rpc_server%s' % i)))
getattr(self, 'mq%s' % i).init_store(self.dirs[2 * i],
self.dirs[2 * i + 1])
thd = threading.Thread(target=getattr(self, 'rpc_server%s' %
i).serve_forever)
thd.setDaemon(True)
thd.start()
self.client = MessageQueueClient(self.nodes)
def put_starts(master=None):
if master is None:
master = ['%s:%s' % (get_ip(), getattr(user_config.master, 'port'))]
print('master:%s' % master)
jobs = client_call(master, 'runnable_jobs')
app_name = ''
for a, j in jobs.items():
if j == "douban movie":
app_name = a
break
if not app_name:
raise Exception('douban movie job has not upload')
nodes = client_call(master, 'list_workers')
addrs = []
default_addr = master.split(':')[0]
for ap, s in nodes:
a, p = ap.split(':')
if a.lower() == 'localhost':
addrs.append('%s:%s' % (default_addr, p))
else:
addrs.append(ap)
mq_client = MessageQueueClient(addrs, app_name)
print('get:%s' % mq_client.get())
urls = []
size = 0
for url in starts:
urls.append(url)
size += 1
if size >= PUTSIZE:
mq_client.put(urls)
size = 0
urls = []
if len(urls) > 0:
mq_client.put(urls)
def __init__(self,
job,
data_dir,
nodes,
local_ip=None,
client=None,
context=None,
copies=1,
force=False):
ctx = context or job.context
master_port = ctx.job.master_port
if local_ip is None:
local_ip = get_ip()
else:
choices_ips = get_ips()
if local_ip not in choices_ips:
raise ValueError('IP address must be one of (%s)' %
','.join(choices_ips))
local = '%s:%s' % (local_ip, master_port)
JobLoader.__init__(self,
job,
data_dir,
local,
context=ctx,
copies=copies,
force=force)
LimitionJobLoader.__init__(self, job, context=ctx)
# check
self.check()
self.nodes = nodes
self.not_registered = self.nodes[:]
self.not_finished = self.nodes[:]
# mq
self.mq_client = MessageQueueClient(self.nodes, copies=copies)
# lock
self.ready_lock = threading.Lock()
self.ready_lock.acquire()
self.finish_lock = threading.Lock()
self.finish_lock.acquire()
# logger
self.logger = get_logger(name='cola_master_%s' % self.job.real_name,
filename=os.path.join(self.root, 'job.log'),
is_master=True)
self.client = client
self.client_handler = None
if self.client is not None:
self.client_handler = add_log_client(self.logger, self.client)
self.init_rpc_server()
self.init_rate_clear()
self.init_logger_server(self.logger)
# register rpc server
self.rpc_server.register_function(self.client_stop, 'client_stop')
self.rpc_server.register_function(self.ready, 'ready')
self.rpc_server.register_function(self.worker_finish, 'worker_finish')
self.rpc_server.register_function(self.complete, 'complete')
self.rpc_server.register_function(self.error, 'error')
self.rpc_server.register_function(self.get_nodes, 'get_nodes')
self.rpc_server.register_function(self.apply, 'apply')
self.rpc_server.register_function(self.require, 'require')
self.rpc_server.register_function(self.stop, 'stop')
self.rpc_server.register_function(self.add_node, 'add_node')
self.rpc_server.register_function(self.remove_node, 'remove_node')
# register signal
signal.signal(signal.SIGINT, self.signal_handler)
signal.signal(signal.SIGTERM, self.signal_handler)
