def _init_function_servers(self):
budget_dir = os.path.join(self.working_dir, 'budget')
budget_cls = BudgetApplyServer if not self.is_multi_process \
else self.manager.budget_server
self.budget_server = budget_cls(budget_dir, self.settings, None,
self.job_name)
if self.rpc_server:
BudgetApplyServer.register_rpc(self.budget_server,
self.rpc_server,
app_name=self.job_name)
self.shutdown_callbacks.append(self.budget_server.shutdown)
counter_dir = os.path.join(self.working_dir, 'counter')
counter_cls = CounterServer if not self.is_multi_process \
else self.manager.counter_server
self.counter_server = counter_cls(counter_dir, self.settings, None,
self.job_name)
if self.rpc_server:
CounterServer.register_rpc(self.counter_server,
self.rpc_server,
app_name=self.job_name)
self.shutdown_callbacks.append(self.counter_server.shutdown)
speed_dir = os.path.join(self.working_dir, 'speed')
speed_cls = SpeedControlServer if not self.is_multi_process \
else self.manager.speed_server
self.speed_server = speed_cls(speed_dir, self.settings, None,
self.job_name, self.counter_server,
self.ctx.ips)
if self.rpc_server:
SpeedControlServer.register_rpc(self.speed_server,
self.rpc_server,
app_name=self.job_name)
self.shutdown_callbacks.append(self.speed_server.shutdown)
def _init_function_servers(self):
budget_dir = os.path.join(self.working_dir, 'budget')
budget_cls = BudgetApplyServer if not self.is_multi_process \
else self.manager.budget_server
self.budget_server = budget_cls(budget_dir, self.settings,
None, self.job_name)
if self.rpc_server:
BudgetApplyServer.register_rpc(self.budget_server, self.rpc_server,
app_name=self.job_name)
self.shutdown_callbacks.append(self.budget_server.shutdown)
counter_dir = os.path.join(self.working_dir, 'counter')
counter_cls = CounterServer if not self.is_multi_process \
else self.manager.counter_server
self.counter_server = counter_cls(counter_dir, self.settings,
None, self.job_name)
if self.rpc_server:
CounterServer.register_rpc(self.counter_server, self.rpc_server,
app_name=self.job_name)
self.shutdown_callbacks.append(self.counter_server.shutdown)
speed_dir = os.path.join(self.working_dir, 'speed')
speed_cls = SpeedControlServer if not self.is_multi_process \
else self.manager.speed_server
self.speed_server = speed_cls(speed_dir, self.settings,
None, self.job_name,
self.counter_server, self.ctx.ips)
if self.rpc_server:
SpeedControlServer.register_rpc(self.speed_server, self.rpc_server,
app_name=self.job_name)
self.shutdown_callbacks.append(self.speed_server.shutdown)
def testSpeedControl(self):
settings = Settings()
counter_server = CounterServer(self.dir_, settings)
self.serv = SpeedControlServer(self.dir_, settings, rpc_server=self.rpc_server,
counter_server=counter_server)
self.cli = SpeedControlClient(self.serv, 'localhost', 0)
self.assertEqual(self.cli.require(10), (10, 0))
self.serv.set_speed(100)
almost = lambda left, right: left[0] == right[0] \
and abs(left[1] - right[1]) < 0.1
self.assertTrue(almost(self.cli.require(100), (100, 0.5)))
self.assertTrue(almost(self.cli.require(1), (0, 0.5)))
self.serv.set_instance_speed(50)
self.serv.calc_spans()
self.serv.reset()
self.assertTrue(almost(self.cli.require(50), (50, 1.1)))
self.assertTrue(almost(self.cli.require(1), (0, 1.1)))
self.serv.set_adaptive(True)
addr = 'localhost#0'
self.serv.counter_server.acc(addr, 'banned_start', 1100)
self.serv.counter_server.acc(addr, 'banned_end', 1500)
self.serv.counter_server.acc(addr, 'normal_start', 1000)
self.serv.counter_server.acc(addr, 'normal_end', 1100)
self.serv.counter_server.acc(addr, 'normal_pages', 200)
self.serv.counter_server.inc(addr, 'pages', 500)
self.serv.counter_server.inc(addr, 'secs', 1000)
self.serv.calc_spans()
self.serv.reset()
self.assertTrue(almost(self.cli.require(24), (24, 0.5)))
self.assertTrue(almost(self.cli.require(1), (0, 0.5)))
def testCounter(self):
self.serv = CounterServer(self.dir_, Settings(),
rpc_server=self.rpc_server)
self.cli1 = CounterClient(self.serv)
self.cli2 = CounterClient('localhost:%s'%self.port)
try:
self.cli1.global_inc('pages', 10)
self.cli2.global_inc('pages', 2)
self.assertEqual(self.cli1.get_global_inc('pages'), 10)
self.assertEqual(self.cli2.get_global_inc('pages'), 2)
self.assertEqual(self.serv.inc_counter.get('global', 'pages', 0), 0)
self.cli1.sync()
self.cli2.sync()
self.assertEqual(self.cli1.get_global_inc('pages'), None)
self.assertEqual(self.cli2.get_global_inc('pages'), None)
self.assertEqual(self.serv.inc_counter.get('global', 'pages'), 12)
self.cli1.local_inc(self.addr, 0, 'pages', 100)
self.assertEqual(self.cli1.get_local_inc(self.addr, 0, 'pages'), 100)
self.cli1.sync()
self.assertEqual(self.serv.inc_counter.get('%s#%s'%(self.addr, 0), 'pages'), 100)
self.cli1.global_acc('normal', 100)
self.cli2.global_acc('normal', 'test')
self.assertEqual(self.cli1.get_global_acc('normal'), [100,])
self.assertEqual(self.cli2.get_global_acc('normal'), ['test',])
self.cli1.sync()
self.cli2.sync()
self.assertEqual(self.cli1.get_global_acc('normal'), None)
self.assertEqual(self.cli2.get_global_acc('normal'), None)
self.assertEqual(self.serv.acc_counter.get('global', 'normal'), [100, 'test'])
self.cli2.local_acc(self.addr, 1, 'normal', 100)
self.assertEqual(self.cli2.get_local_acc(self.addr, 1, 'normal'), [100, ])
self.cli2.sync()
self.assertEqual(self.serv.acc_counter.get('%s#%s'%(self.addr, 1), 'normal'), [100, ])
finally:
self.serv.shutdown()
def testBudgetApply(self):
self.serv = BudgetApplyServer(self.dir_, Settings(), rpc_server=self.rpc_server)
self.cli1 = BudgetApplyClient(self.serv)
self.cli2 = BudgetApplyClient('localhost:%s'%self.port)
try:
self.serv.set_budgets(90)
self.assertEqual(self.cli1.apply(50), 50)
self.assertEqual(self.cli2.apply(50), 40)
self.cli1.finish(50)
self.assertEqual(50, self.serv.finished)
self.cli2.finish(50)
self.assertEqual(90, self.serv.finished)
self.cli1.error(10)
self.assertEqual(90, self.serv.applied)
self.serv.finished = 0
self.cli2.error(10)
self.assertEqual(80, self.serv.applied)
finally:
self.serv.shutdown()
def _init_speed_server(self):
speed_dir = os.path.join(self.working_dir, 'speed')
self.speed_server = SpeedControlServer(speed_dir,
self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
class JobMaster(object):
def __init__(self, ctx, job_name, job_desc, workers):
self.working_dir = os.path.join(ctx.working_dir, 'master', 'tracker',
job_name)
if not os.path.exists(self.working_dir):
os.makedirs(self.working_dir)
self.job_name = job_name
self.job_desc = job_desc
self.settings = job_desc.settings
self.rpc_server = ctx.master_rpc_server
self.inited = False
self.lock = threading.Lock()
self.init()
self.workers = workers
def _init_counter_server(self):
counter_dir = os.path.join(self.working_dir, 'counter')
self.counter_server = CounterServer(counter_dir,
self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
def _init_budget_server(self):
budget_dir = os.path.join(self.working_dir, 'budget')
self.budget_server = BudgetApplyServer(budget_dir,
self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
def _init_speed_server(self):
speed_dir = os.path.join(self.working_dir, 'speed')
self.speed_server = SpeedControlServer(speed_dir,
self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
def init(self):
with self.lock:
if self.inited:
return
self._init_counter_server()
self._init_budget_server()
self._init_speed_server()
self.inited = True
def remove_worker(self, worker):
if worker not in self.workers:
return
# rpc call the other workers to remove this worker
self.workers.remove(worker)
for node in self.workers:
client_call(node, 'remove_node', worker)
def add_worker(self, worker):
if worker in self.workers:
return
# rpc call the other workers to add this worker
for node in self.workers:
client_call(node, 'add_node', worker)
self.workers.append(worker)
def has_worker(self, worker):
return worker in self.workers
def shutdown(self):
with self.lock:
if not self.inited:
return
self.counter_server.shutdown()
self.budget_server.shutdown()
self.speed_server.shutdown()
self.inited = False
def _init_speed_server(self):
speed_dir = os.path.join(self.working_dir, 'speed')
self.speed_server = SpeedControlServer(speed_dir, self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
class JobMaster(object):
def __init__(self, ctx, job_name, job_desc, workers):
self.working_dir = os.path.join(ctx.working_dir, 'master',
'tracker', job_name)
if not os.path.exists(self.working_dir):
os.makedirs(self.working_dir)
self.job_name = job_name
self.job_desc = job_desc
self.settings = job_desc.settings
self.rpc_server = ctx.master_rpc_server
self.inited = False
self.lock = threading.Lock()
self.init()
self.workers = workers
def _init_counter_server(self):
counter_dir = os.path.join(self.working_dir, 'counter')
self.counter_server = CounterServer(counter_dir, self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
def _init_budget_server(self):
budget_dir = os.path.join(self.working_dir, 'budget')
self.budget_server = BudgetApplyServer(budget_dir, self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
def _init_speed_server(self):
speed_dir = os.path.join(self.working_dir, 'speed')
self.speed_server = SpeedControlServer(speed_dir, self.settings,
rpc_server=self.rpc_server,
app_name=self.job_name)
def init(self):
with self.lock:
if self.inited:
return
self._init_counter_server()
self._init_budget_server()
self._init_speed_server()
self.inited = True
def remove_worker(self, worker):
if worker not in self.workers:
return
# rpc call the other workers to remove this worker
self.workers.remove(worker)
for node in self.workers:
client_call(node, 'remove_node', worker)
def add_worker(self, worker):
if worker in self.workers:
return
# rpc call the other workers to add this worker
for node in self.workers:
client_call(node, 'add_node', worker)
self.workers.append(worker)
def has_worker(self, worker):
return worker in self.workers
def shutdown(self):
with self.lock:
if not self.inited:
return
self.counter_server.shutdown()
self.budget_server.shutdown()
self.speed_server.shutdown()
self.inited = False
class Test(unittest.TestCase):
def setUp(self):
self.port = random.randint(10000, 30000)
self.addr = 'localhost:%s' % self.port
self.rpc_server = ColaRPCServer(('localhost', self.port))
thd = threading.Thread(target=self.rpc_server.serve_forever)
thd.setDaemon(True)
thd.start()
self.dir_ = tempfile.mkdtemp()
def tearDown(self):
try:
self.rpc_server.shutdown()
finally:
shutil.rmtree(self.dir_)
def testBudgetApply(self):
self.serv = BudgetApplyServer(self.dir_, Settings(), rpc_server=self.rpc_server)
self.cli1 = BudgetApplyClient(self.serv)
self.cli2 = BudgetApplyClient('localhost:%s'%self.port)
try:
self.serv.set_budgets(90)
self.assertEqual(self.cli1.apply(50), 50)
self.assertEqual(self.cli2.apply(50), 40)
self.cli1.finish(50)
self.assertEqual(50, self.serv.finished)
self.cli2.finish(50)
self.assertEqual(90, self.serv.finished)
self.cli1.error(10)
self.assertEqual(90, self.serv.applied)
self.serv.finished = 0
self.cli2.error(10)
self.assertEqual(80, self.serv.applied)
finally:
self.serv.shutdown()
def testCounter(self):
self.serv = CounterServer(self.dir_, Settings(),
rpc_server=self.rpc_server)
self.cli1 = CounterClient(self.serv)
self.cli2 = CounterClient('localhost:%s'%self.port)
try:
self.cli1.global_inc('pages', 10)
self.cli2.global_inc('pages', 2)
self.assertEqual(self.cli1.get_global_inc('pages'), 10)
self.assertEqual(self.cli2.get_global_inc('pages'), 2)
self.assertEqual(self.serv.inc_counter.get('global', 'pages', 0), 0)
self.cli1.sync()
self.cli2.sync()
self.assertEqual(self.cli1.get_global_inc('pages'), None)
self.assertEqual(self.cli2.get_global_inc('pages'), None)
self.assertEqual(self.serv.inc_counter.get('global', 'pages'), 12)
self.cli1.local_inc(self.addr, 0, 'pages', 100)
self.assertEqual(self.cli1.get_local_inc(self.addr, 0, 'pages'), 100)
self.cli1.sync()
self.assertEqual(self.serv.inc_counter.get('%s#%s'%(self.addr, 0), 'pages'), 100)
self.cli1.global_acc('normal', 100)
self.cli2.global_acc('normal', 'test')
self.assertEqual(self.cli1.get_global_acc('normal'), [100,])
self.assertEqual(self.cli2.get_global_acc('normal'), ['test',])
self.cli1.sync()
self.cli2.sync()
self.assertEqual(self.cli1.get_global_acc('normal'), None)
self.assertEqual(self.cli2.get_global_acc('normal'), None)
self.assertEqual(self.serv.acc_counter.get('global', 'normal'), [100, 'test'])
self.cli2.local_acc(self.addr, 1, 'normal', 100)
self.assertEqual(self.cli2.get_local_acc(self.addr, 1, 'normal'), [100, ])
self.cli2.sync()
self.assertEqual(self.serv.acc_counter.get('%s#%s'%(self.addr, 1), 'normal'), [100, ])
finally:
self.serv.shutdown()
def testSpeedControl(self):
settings = Settings()
counter_server = CounterServer(self.dir_, settings)
self.serv = SpeedControlServer(self.dir_, settings, rpc_server=self.rpc_server,
counter_server=counter_server)
self.cli = SpeedControlClient(self.serv, 'localhost', 0)
self.assertEqual(self.cli.require(10), (10, 0))
self.serv.set_speed(100)
almost = lambda left, right: left[0] == right[0] \
and abs(left[1] - right[1]) < 0.1
self.assertTrue(almost(self.cli.require(100), (100, 0.5)))
self.assertTrue(almost(self.cli.require(1), (0, 0.5)))
self.serv.set_instance_speed(50)
self.serv.calc_spans()
self.serv.reset()
self.assertTrue(almost(self.cli.require(50), (50, 1.1)))
self.assertTrue(almost(self.cli.require(1), (0, 1.1)))
self.serv.set_adaptive(True)
addr = 'localhost#0'
self.serv.counter_server.acc(addr, 'banned_start', 1100)
self.serv.counter_server.acc(addr, 'banned_end', 1500)
self.serv.counter_server.acc(addr, 'normal_start', 1000)
self.serv.counter_server.acc(addr, 'normal_end', 1100)
self.serv.counter_server.acc(addr, 'normal_pages', 200)
self.serv.counter_server.inc(addr, 'pages', 500)
self.serv.counter_server.inc(addr, 'secs', 1000)
self.serv.calc_spans()
self.serv.reset()
self.assertTrue(almost(self.cli.require(24), (24, 0.5)))
self.assertTrue(almost(self.cli.require(1), (0, 0.5)))