async def run(self, args):
self.seq = args.seq
while get_cluster().is_running():
print(self.seq, 'step 1')
await sleep(3)
# will continue executing about 15 steps when pressing ctrl-c
for _ in range(100):
print('stopping', get_cluster().is_stopping())
print(self.seq, 'step 2')
await sleep(1)
print(self.seq, 'end')
async def run(self, args: Namespace) -> None:
ticket = get_ticket()
if ticket.language != 'python3':
print('language must be python3', file=sys.stderr)
sys.exit(1)
mod = import_module(args.module)
if hasattr(mod, 'Handler'):
handler = mod.Handler()
else:
handler = BaseHandler()
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGINT, self.handle_stop_sig, handler)
loop.add_signal_handler(signal.SIGTERM, self.handle_stop_sig, handler)
loop.set_exception_handler(coroutine_exc_handler)
parser = argparse.ArgumentParser(prog='bbox.py run')
handler.add_arguments(parser)
sub_args = parser.parse_args(args.task_params)
try:
await get_cluster().start()
r = await handler.run(sub_args)
if r is not None:
logger.debug('task return %s', r)
finally:
handler.shutdown()
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
async def ensure_clients(self, srv):
if has_service(srv):
return
agent = get_cluster()
boxes = agent.route[srv]
# connect at most n concurrent connections
for bind in sorted(boxes)[:self.max_concurrency]:
if bind not in self.pool:
# add box to pool
box = agent.boxes[bind]
client = WebSocketClient(bind)
self.pool[bind] = client
for bind, client in list(self.pool.items()):
if bind not in agent.boxes:
logger.warn('remove box %s', bind)
# remove box due to server done
client.close()
del self.pool[bind]
for _ in range(30):
c = self.get_client(srv, policy=self.FIRST)
if c:
return
await asyncio.sleep(0.01)
async def create_consumer(request, consumer, reuse):
'''
createConsumer(consumer, reuse)
create a consumer for test, note that this methods is *ONLY* used for testing. Production consumer can be created using `bbox.py run aiobbox.contrib.consumer.create_consumer <name>`
'''
if not testing.test_mode:
raise ServiceError('access denied')
if (not isinstance(consumer, str) or not re.match(r'\w+$', consumer)):
raise ServiceError('invalid consumer')
cfg = get_sharedconfig()
coptions = cfg.get('consumers', consumer)
#if coptions:
# raise ServiceError('consumer already exist',
# msg='already exist {}'.format(coptions))
if coptions and reuse:
return {'consumer': consumer, 'secret': coptions['secret']}
coptions = {}
coptions['secret'] = uuid.uuid4().hex
coptions['seed'] = ssl.RAND_bytes(256).hex()
c = get_cluster()
await c.set_config('consumers', consumer, coptions)
# TODO: limit the consumer size
return {'consumer': consumer, 'secret': coptions['secret']}
async def run(self, args):
consumer = args.consumer
if not consumer:
raise Exception('void consumer')
if not re.match(r'\w+$', consumer):
raise Exception('invalid consumer')
cfg = get_sharedconfig()
coptions = cfg.get('consumers', consumer)
if coptions:
print('consumer {} already exist, secret is {}'.format(
consumer, coptions['secret']))
return
coptions = {}
coptions['secret'] = uuid.uuid4().hex
coptions['seed'] = ssl.RAND_bytes(256).hex()
c = get_cluster()
await c.set_config('consumers', consumer, coptions)
# TODO: limit the consumer size
print('secret for', consumer, coptions['secret'])
async def run(self, args):
srv, method = args.srv_method.split('::')
ps = [guess_json(p) for p in args.param]
if args.dispatch_policy == 'random':
bbox_client.pool.policy = bbox_client.pool.RANDOM
try:
await get_cluster().start()
for i in range(args.ntimes):
r = await bbox_client.pool.request(srv,
method,
*ps,
retry=args.retry)
if args.pp:
print(json_pp(r))
else:
print(json_to_str(r))
if (args.stack and r.get('error')
and isinstance(r['error'], dict)
and r['error'].get('stack')):
print('\nerror stack:')
print(r['error']['stack'])
if i >= args.ntimes - 1:
break
await asyncio.sleep(args.interval)
finally:
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
async def run(self, args):
cfg = get_ticket()
if cfg.language != 'python3':
print('language must be python3', file=sys.stderr)
sys.exit(1)
mod = import_module(args.module)
if hasattr(mod, 'Handler'):
handler = mod.Handler()
else:
handler = BaseHandler()
parser = argparse.ArgumentParser(prog='bbox.py run')
handler.add_arguments(parser)
sub_args = parser.parse_args(args.task_params)
try:
await get_cluster().start()
r = await handler.run(sub_args)
if r:
logger.debug('task return %s', r)
finally:
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
async def cluster_info(args) -> None:
ticket = get_ticket()
info = {
'etcd': ticket.etcd,
'prefix': ticket.prefix,
'boxes': get_cluster().boxes
}
print(json_pp(info))
async def cluster_info(args):
cfg = get_ticket()
info = {
'etcd': cfg.etcd,
'prefix': cfg.prefix,
'boxes': get_cluster().boxes
}
print(json_pp(info))
def get_client_count(self, srv):
cc = get_cluster()
cnt = 0
for bind in cc.route[srv]:
client = self.pool.get(bind)
if client and client.connected:
cnt += 1
return cnt
def handle_stop_sig(self, handler: BaseHandler) -> None:
try:
logger.debug('sigint met, the handle %s should stop lately',
handler)
get_cluster().stop()
handler.cont = False
wakeup_sleep_tasks()
loop = asyncio.get_event_loop()
loop.remove_signal_handler(signal.SIGINT)
loop.remove_signal_handler(signal.SIGTERM)
exit_after = int(os.getenv('BBOX_TASK_EXIT_WAIT', 10))
loop.call_later(
exit_after,
sys_exit) # force exit10 or env defined seconds later
except:
logger.error('error on handle sigint', exc_info=True)
raise
def __init__(self, connect):
c = get_cluster()
box = c.boxes[connect]
self.ssl_prefix = box['ssl']
if self.ssl_prefix:
self.url_prefix = 'https://' + connect
else:
self.url_prefix = 'http://' + connect
ssl_context = get_cert_ssl_context(self.ssl_prefix)
conn = aiohttp.TCPConnector(ssl_context=ssl_context)
self.session = aiohttp.ClientSession(connector=conn)
async def take_jobs(self, whitelist:Optional[List[str]]=None, parallel:bool=False) -> None:
self.whitelist = whitelist
redis_pool = await self.get_redis_pool()
while get_cluster().is_running():
key, body = await redis_pool.execute(
'BRPOP', self.req_key, 0)
logging.debug('brpoped %s %s', key, body)
body = json.loads(body)
if parallel:
asyncio.ensure_future(self.handle_req(body))
else:
await self.handle_req(body)
async def run(self, args: Namespace) -> None:
await get_cluster().start()
func = getattr(args, 'func', None)
if func is None:
print('bbox.py cluster -h')
else:
try:
await args.func(args)
finally:
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
async def run(self, args):
await get_cluster().start()
func = getattr(args, 'func', None)
if func:
try:
await args.func(args)
finally:
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
else:
print('type bbox.py config -h')
def report_box_failure(bind:str) -> Dict[str, Any]:
meta = {
'box_fail': {
'type': 'gauge',
'help': 'box cannot connect'
}
}
c = get_cluster()
box = c.boxes.get(bind)
labels = {'bind': bind}
if box:
labels['box'] = box['boxid']
lines = [('box_fail', labels, 1)]
return {
'meta': meta,
'lines': lines
}
async def run(self, args):
cluster = get_cluster()
await cluster.start()
if args.skipheaders.lower() not in ('yes', 'true', '1'):
headers = ['srv', 'bind', 'boxid', 'start', 'ssl_prefix']
print('\t'.join(headers))
print('\t'.join(['-' * len(h) for h in headers]))
now = datetime.utcnow()
for srv, boxes in cluster.route.items():
for bind in boxes:
row = [srv, bind]
box = cluster.boxes.get(bind, {})
row.append(box.get('boxid', '-'))
start_time = box.get('start_time')
row.append(start_time or '-')
row.append(box.get('ssl') or '-')
print('\t'.join(row))
async def handle_metrics(request):
# check bearer token
if bearer_token:
if (request.headers.get('Authorization')
!= 'Bearer {}'.format(bearer_token)):
raise web.HTTPUnauthorized()
c = get_cluster()
with ClientSession() as session:
if collect_localbox:
fns = [get_box_metrics(bind)
for bind in c.get_local_boxes()]
else:
fns = [get_box_metrics(bind)
for bind in c.boxes.keys()]
if fns:
res = await asyncio.gather(*fns)
else:
res = []
if export_cluster:
res.append(collect_cluster_metrics())
meta = {}
lines = []
meta_lines = []
for resp in res:
meta.update(resp['meta'])
for name, labels, v in resp['lines']:
d = ', '.join('{}="{}"'.format(lname, lvalue)
for lname, lvalue in labels.items())
d = '{' + d + '}'
lines.append('{} {} {}'.format(name, d, v))
for name, define in meta.items():
meta_lines.append('# HELP {} {}'.format(
name, define['help']))
meta_lines.append('# TYPE {} {}'.format(
name, define['type']))
meta_lines.append('')
headers = {'Content-Type': 'text/plain'}
return web.Response(text='\n'.join(meta_lines + lines + ['']),
headers=headers)
def get_client(self, srv, policy=None, boxid=None):
policy = policy or self.policy
clients = []
cc = get_cluster()
for bind in cc.route[srv]:
client = self.pool.get(bind)
if client and client.connected:
if boxid:
box = cc.boxes.get(bind)
if box.boxid != boxid:
continue
if policy == self.FIRST:
return client
else:
assert policy == self.RANDOM
clients.append(client)
if clients:
return random.choice(clients)
def __init__(self, connect):
c = get_cluster()
box = c.boxes[connect]
self.ssl_prefix = box['ssl']
if self.ssl_prefix:
self.url_prefix = 'wss://' + connect
else:
self.url_prefix = 'ws://' + connect
ssl_context = get_cert_ssl_context(self.ssl_prefix)
conn = aiohttp.TCPConnector(ssl_context=ssl_context)
self.session = aiohttp.ClientSession(connector=conn)
self.waiters = {}
self.ws = None
self.notify_channel = None
self.cont = True
asyncio.ensure_future(self.connect_wait())
def collect_cluster_metrics() -> Dict[str, Any]:
cluster = get_cluster()
meta = {
'service_boxes': {
'type': 'gauge',
'help': 'The count of boxes for each service'
}
}
lines = []
for srv_name, boxes in cluster.route.items():
lines.append(
('service_boxes',
{'srv': srv_name},
len(boxes)))
return {
'meta': meta,
'lines': lines
}
def get_client(self, srv_name: str, policy: int=None, boxid: str=None) -> Optional[HttpClient]:
policy = policy or self.policy
connects = []
cc = get_cluster()
for bind in cc.route[srv_name]:
if boxid:
box = cc.boxes.get(bind)
if box.boxid != boxid:
continue
if policy == self.FIRST:
connects.append(bind)
break
else:
assert policy == self.RANDOM
connects.append(bind)
if connects:
connect = random.choice(connects)
return HttpClient(connect, expect='json')
return None
async def run(self, args):
if get_ticket().language != 'python3':
print('language must be python3', file=sys.stderr)
sys.exit(1)
cors = []
handlers = []
for spec in args.taskspec:
args = shlex.split(spec)
module, task_params = args[0], args[1:]
mod = import_module(module)
if hasattr(mod, 'Handler'):
handler = mod.Handler()
else:
handler = BaseHandler()
parser = argparse.ArgumentParser(
prog='bbox.py mrun {}'.format(module))
handler.add_arguments(parser)
sub_args = parser.parse_args(task_params)
handlers.append(handler)
cors.append(self.run_handler(handler, sub_args))
loop = asyncio.get_event_loop()
loop.add_signal_handler(signal.SIGINT, self.handle_stop_sig, handlers)
loop.add_signal_handler(signal.SIGTERM, self.handle_stop_sig, handlers)
loop.set_exception_handler(coroutine_exc_handler)
try:
await get_cluster().start()
await asyncio.gather(*cors)
finally:
handler.shutdown()
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
async def run(self, args):
try:
await get_cluster().start()
resp = await bbox_client.pool.request(args.srv_name, '__doc__')
if not resp.get('result'):
return
r = resp['result']
if args.nameonly:
print(r['name'])
print('methods:')
for method in r['methods']:
print(' {}'.format(method['name']))
elif args.format in ('markdown', 'md'):
print_markdown(r)
else:
print_text(r)
finally:
c = get_cluster()
c.cont = False
await asyncio.sleep(0.1)
c.close()
