def run(self, args, manage_args):
environment = get_environment(args.env_name)
celery_processes = environment.app_processes_config.celery_processes
by_queue = defaultdict(lambda: {'num_workers': 0, 'concurrency': 0, 'pooling': set(), 'worker_hosts': set()})
for host, queues in celery_processes.items():
for queue_name, options in queues.items():
queue = by_queue[queue_name]
queue['num_workers'] += options.num_workers
queue['concurrency'] += options.concurrency * options.num_workers
queue['pooling'].add(options.pooling)
queue['worker_hosts'].add(host)
headers = ['Pooling', 'Worker Queues', 'Processes', 'Concurrency', 'Avg Concurrency per worker']
if args.show_workers:
headers.append('Worker Hosts')
rows = []
for queue_name, stats in sorted(by_queue.items(), key=itemgetter(0)):
workers = stats['num_workers']
concurrency_ = stats['concurrency']
row = [list(stats['pooling'])[0], '`{}`'.format(queue_name), workers, concurrency_, concurrency_ // workers]
if args.show_workers:
worker_hosts = stats['worker_hosts']
row.append(','.join(sorted([get_machine_alias(environment, worker_host) for worker_host in worker_hosts])))
rows.append(row)
print_table(headers, rows, args.csv)
def test_get_machine_alias():
env = get_environment('small_cluster')
all_hosts = set(itertools.chain.from_iterable(env.groups.values()))
assert_equal(all_hosts, {'172.19.3.0', '172.19.3.1', '172.19.3.2', '172.19.3.3'})
aliases = set([get_machine_alias(env, host) for host in all_hosts])
assert_equal(aliases, {'demo_server0', 'demo_server1', 'demo_server2', 'demo_server3'})
def get_couch_config(environment, nodes=None, port=15984, local_port=15986):
couch_nodes = nodes or environment.groups['couchdb2']
config = Config(
control_node_ip=couch_nodes[0],
control_node_port=port,
control_node_local_port=local_port,
username=environment.get_secret('COUCH_USERNAME'),
aliases={
'couchdb@{}'.format(node): get_machine_alias(environment, node) for node in couch_nodes
}
)
config.set_password(environment.get_secret('COUCH_PASSWORD'))
return config
def get_couch_config(environment, nodes=None):
couch_nodes = nodes or environment.groups['couchdb2']
config = Config(control_node_ip=couch_nodes[0],
control_node_port=15984,
control_node_local_port=15986,
username=environment.get_vault_var(
'localsettings_private.COUCH_USERNAME'),
aliases={
'couchdb@{}'.format(node):
get_machine_alias(environment, node)
for node in couch_nodes
})
config.set_password(
environment.get_vault_var('localsettings_private.COUCH_PASSWORD'))
return config
def get_couch_config(environment, nodes=None, port=15984, local_port=15986, couchdb_version=None):
couch_nodes = nodes or environment.groups['couchdb2']
if couchdb_version is None:
couchdb_version = environment.public_vars.get('couchdb_version', '2.3.1')
if LooseVersion(couchdb_version) >= LooseVersion('3.0.0'):
local_port = port
config = Config(
control_node_ip=couch_nodes[0],
control_node_port=port,
control_node_local_port=local_port,
couchdb_version=couchdb_version,
username=environment.get_secret('COUCH_USERNAME'),
aliases={
'couchdb@{}'.format(node): get_machine_alias(environment, node) for node in couch_nodes
}
)
config.set_password(environment.get_secret('COUCH_PASSWORD'))
return config
def run(self, args, manage_args):
environment = get_environment(args.env_name)
celery_processes = environment.app_processes_config.celery_processes
by_queue = defaultdict(
lambda: {
'num_workers': 0,
'concurrency': 0,
'pooling': set(),
'worker_hosts': set()
})
for host, queues in celery_processes.items():
for queue_name, options in queues.items():
queue = by_queue[queue_name]
queue['num_workers'] += options.num_workers
queue[
'concurrency'] += options.concurrency * options.num_workers
queue['pooling'].add(options.pooling)
queue['worker_hosts'].add(host)
max_name_len = max([len(name) for name in by_queue])
template = "{{:<8}} | {{:<{}}} | {{:<12}} | {{:<12}} | {{:<12}} | {{:<12}}".format(
max_name_len + 2)
print(
template.format('Pooling', 'Worker Queues', 'Processes',
'Concurrency', 'Avg Concurrency per worker',
'Worker Hosts'))
print(
template.format('-------', '-------------', '---------',
'-----------', '--------------------------',
'------------'))
for queue_name, stats in sorted(by_queue.items(), key=itemgetter(0)):
workers = stats['num_workers']
concurrency_ = stats['concurrency']
worker_hosts = stats['worker_hosts']
print(
template.format(
list(stats['pooling'])[0], '`{}`'.format(queue_name),
workers, concurrency_, concurrency_ // workers, ','.join(
sorted([
get_machine_alias(environment, worker_host)
for worker_host in worker_hosts
]))))