async def get_nodes_info(request: Request) -> list: client = get_client(request) info = await client.nodes.stats(metric='jvm,os,fs') result = [] for node_id, node in info['nodes'].items(): result.append({ "name": node["name"], "isMaster": False, "ip": node.get("ip", "no-ip"), "roles": node["roles"], "id": node_id, "metrics": { # Reduce precision in order to reduce render loops in UI "CPUPercent": int(node['os']['cpu']['percent']), "heapPercent": int(node['jvm']['mem']['heap_used_percent']), "load1Percent": int(float(node['os']['cpu']['load_average']['1m']) / 4), # TODO: Count CPUs "diskPercent": int(node['fs']['total']['available_in_bytes'] * 100 / node['fs']['total']['total_in_bytes']) } }) return result
async def get_cluster_info(request: Request, cluster_name: str) -> HTTPResponse: client = get_client(None, cluster_name) await client.ping() response = await client.cluster.stats() try: jvm = 'mixed' if len( response['nodes']['jvm']['versions'] ) > 1 else response['nodes']['jvm']['versions'][0]['vm_name'] # For some reasone es6.3.1 is missing jvm version except KeyError: jvm = 'unknown' return json({ "name": response['cluster_name'], "versions": response['nodes']['versions'], "jvmName": jvm, "numNodes": response['_nodes']['total'], "status": response['status'], "docCount": response['indices']['docs']['count'], "storeSizeInBytes": response['indices']['store']['size_in_bytes'] })
async def node_info(request: Request, node_id: str) -> HTTPResponse: client = get_client(request) info = (await client.nodes.info(node_id=node_id))['nodes'][node_id] return json({ 'OS': { "Ip Address": info['ip'], "Roles": ", ".join(info['roles']), "Logs Path": info['settings']['path']['logs'], "Home Path": info['settings']['path']['home'], "Available Processors": info['os']['available_processors'], "Allocated Processors": info['os']['allocated_processors'] }, "Thread Pools": { "search": "{x[queue_size]} / {x[size]}".format(x=info['thread_pool']['search']), "write": "{x[queue_size]} / {x[size]}".format(x=info['thread_pool']['write']), "get": "{x[queue_size]} / {x[size]}".format(x=info['thread_pool']['get']), "snapshot": "{x[queue_size]} / {x[max]}".format(x=info['thread_pool']['snapshot']), "management": "{x[queue_size]} / {x[max]}".format(x=info['thread_pool']['management']), "force_merge": "{x[queue_size]} / {x[size]}".format(x=info['thread_pool']['force_merge']), }, "JVM": { "pid": info['jvm']['pid'], "version": info['jvm']['version'], "vmName": info['jvm']['vm_name'], "vmVendor": info['jvm']['vm_vendor'], "startTimeInMillis": info['jvm']['start_time_in_millis'], "heapMaxInMb": info['jvm']['mem']['heap_max_in_bytes']/1024/1024 } })
async def list_repos(request: Request) -> HTTPResponse: client = get_client(request) repos = [] resp = await client.snapshot.get_repository() for repo_id, repo_type in resp.items(): repos.append({"id": repo_id, "type": repo_type}) return json(repos)
async def get_body(request: Request, index: str) -> HTTPResponse: client = get_client(request) current_index_settings = (await client.indices.get(index))[index] if 'index' in current_index_settings['settings']: current_index_settings['settings']['index'].pop('creation_date', None) current_index_settings['settings']['index'].pop('uuid', None) current_index_settings['settings']['index'].pop('version', None) current_index_settings['settings']['index'].pop('provided_name', None) return json(current_index_settings)
async def set_allocation(request: Request, operation: str) -> HTTPResponse: assert operation in {"all", "primaries", "new_primaries", "none"} client = get_client(request) response = await client.cluster.put_settings( body={"transient": { "cluster.routing.allocation.enable": operation }}) assert response.get('acknowledged') is True return json({"status": "ok"})
async def indices_stats(request: Request) -> HTTPResponse: client = get_client(request) indices, aliases, shards, nodes, cluster_info = await gather( client.cat.indices(format='json'), get_index_aliases(request), get_shards_info(request), get_nodes_info(request), get_cluster_info(request)) shards, relocation_progress = shards cluster_info["numOfIndices"] = len(indices) indices_per_node = defaultdict(lambda: defaultdict(lambda: { 'replicas': [], 'primaries': [] })) unassigned_shards = defaultdict(lambda: {'replicas': [], 'primaries': []}) for shard in shards: if shard['prirep'] == 'p': shard_type = 'primaries' elif shard['prirep'] == 'r': shard_type = 'replicas' else: raise RuntimeError('Unknown shard type %s' % shard['prirep']) data = { "shard": int(shard['shard']), "state": shard['state'], } node = shard['node'] if shard['state'] == 'UNASSIGNED': unassigned_shards[shard['index']][shard_type] = sorted( unassigned_shards[shard['index']][shard_type] + [data], key=lambda x: x['shard']) if shard['state'] == 'RELOCATING': from_node, to_node = node.split(' -> ') node = from_node data['fromNode'] = to_node.split()[2] data['progress'] = relocation_progress.get( (shard['index'], shard['shard']), 0) if shard['state'] == 'INITIALIZING': data['progress'] = relocation_progress.get( (shard['index'], shard['shard']), 0) indices_per_node[node][shard['index']][shard_type].append(data) indices_per_node[node][shard['index']][shard_type].sort( key=lambda x: x['shard']) for node in nodes: node['indices'] = dict(sorted(indices_per_node[node["name"]].items())) indices = dict( sorted([(x['index'], format_index_data(x, aliases)) for x in indices])) for index in indices: if index in unassigned_shards: indices[index]['unassignedShards'] = unassigned_shards[index] return json({ "nodes": sorted(nodes, key=lambda x: x["name"]), "indices": indices, "cluster": cluster_info, })
async def dynamic_settings(request: Request, index: str) -> HTTPResponse: client = get_client(request) all_settings = await client.indices.get_settings(index=index, flat_settings=True, include_defaults=True) chained = ChainMap(*all_settings[index].values()) sections = deepcopy(get_index_settings_docs()) for section in sections: for setting in section['settings']: if setting['name'] in chained: setting['value'] = chained[setting['name']] return json(sections)
async def close_index(request: Request) -> HTTPResponse: client = get_client(request) body = request.json['body'] method = request.json['method'] path = request.json['path'] try: resp = await client.transport.perform_request(method, path, body=body) except TransportError as e: return format_es_exception(e) return json(resp)
async def list_tasks(request: Request) -> HTTPResponse: result = [] client = get_client(request) tasks = await client.tasks.list(detailed=True, group_by='parents') for task_id, task in tasks['tasks'].items(): formatted = { "taskId": task_id, "action": task['action'], "running_time_in_ms": task["running_time_in_nanos"] / 1_000_000, "cancellable": task["cancellable"], "nodeName": task['node'], }
async def reroute_shard(request: Request) -> HTTPResponse: client = get_client(request) node = request.json['node'] shards = request.json['shards'] await client.cluster.reroute( body={ "commands": [{ "move": { "index": shard['index'], "shard": shard['id'], "from_node": shard['nodeName'], "to_node": node } } for shard in shards] }) return json({"status": "ok"})
async def get_shards_info(request: Request) -> tuple: client = get_client(request) shards = await client.cat.shards(format='json') relocating_indices = list( {shard['index'] for shard in shards if shard['state'] == 'RELOCATING'}) recovery = await client.cat.recovery(index=relocating_indices, format='json') relocation_progress = { (recovery_data["index"], recovery_data["shard"]): int( float(recovery_data["bytes_recovered"]) * 100 / (int(recovery_data["bytes_total"]) or 1)) for recovery_data in recovery if recovery_data["stage"] != "done" and recovery_data["bytes_total"] } return shards, relocation_progress
async def get_cluster_info(request: Request) -> dict: client = get_client(request) [info], [docs], settings = await gather( client.cat.health(format='json'), client.cat.count(format='json'), client.cluster.get_settings(flat_settings=True)) return { "relocatingShards": int(info['relo']), "initializingShards": int(info['init']), "unassignedShards": int(info['unassign']), "numOfPrimaryShards": int(info['pri']), "numOfReplicaShards": int(info["shards"]) - int(info['pri']), "numberOfNodes": int(info['node.total']), "numberOfDocs": int(docs['count']), "clusterName": info['cluster'], "clusterStatus": info['status'], "settings": { "allocation": settings['transient'].get('cluster.routing.allocation.enable', 'all') }, }
async def delete_index(request: Request, index: str) -> HTTPResponse: client = get_client(request) await client.indices.delete(index=index) return json({"status": "ok"})
async def clear_cache(request: Request, index: str) -> HTTPResponse: client = get_client(request) await client.indices.clear_cache(index=index, allow_no_indices=False) return json({"status": "ok"})
async def cancle_task(request: Request) -> HTTPResponse: client = get_client(request) task_id = request.json['taskId'] await client.tasks.cancle(task_id=task_id) return json({"status": "ok"})
async def create_template(request: Request, template: str) -> HTTPResponse: client = get_client(request) await client.indices.put_template(name=template, body=request.json, create=True) return json("ok")
async def index_settings(request: Request, index: str) -> HTTPResponse: client = get_client(request) return json(await client.indices.get_settings(index=index, flat_settings=True))
async def set_index_settings(request: Request, index: str) -> HTTPResponse: client = get_client(request) return json(await client.indices.put_settings(dict(request.json), index))
async def list_snaphosts(request: Request, repo: str) -> HTTPResponse: client = get_client(request) resp = await client.snapshot.get(repo, '_all') return json(resp['snapshots'])
async def list_templates(request: Request) -> HTTPResponse: client = get_client(request) templates = await client.indices.get_template() return json([dict(name=x, **templates[x]) for x in templates])
async def open_index(request: Request, index: str) -> HTTPResponse: client = get_client(request) await client.indices.open(index=index, expand_wildcards='none') return json({"status": "ok"})
async def delete_template(request: Request, template: str) -> HTTPResponse: client = get_client(request) await client.indices.delete_template(template) return json("ok")
async def graveyard(request: Request) -> HTTPResponse: client = get_client(request) resp = await client.cluster.state(metric='metadata') return json(resp['metadata']['index-graveyard']['tombstones'])
async def restore(request: Request, repo: str, snapshot: str) -> HTTPResponse: client = get_client(request) response = await client.snapshot.restore(repo, snapshot, request.json) return json(response)
async def get_mapping(request: Request, index: str) -> HTTPResponse: client = get_client(request) return json(await client.indices.get_mapping(index=index))
def send_test(client, data): target_client = connections.get_client(data['clientIndex']) target_client.send_message({ 'endpoint': 'receive-test', 'message': 'hello world', })
async def head_index(request: Request, index: str) -> HTTPResponse: client = get_client(request) content = await client.search(index=index) return json(x['_source'] for x in content['hits']['hits'])
async def index_stats(request: Request, index: str) -> HTTPResponse: client = get_client(request) return json(await client.indices.stats(index=index))
async def node_stats(request: Request, node_id: str) -> HTTPResponse: client = get_client(request) stats = (await client.nodes.stats(node_id=node_id))['nodes'][node_id] return json({ 'threadPool': stats['thread_pool'] })