def get_elasticsearch_index_samples(elasticsearch_index):
es_client = get_es_client()
index_metadata = get_index_metadata(elasticsearch_index, es_client).get(elasticsearch_index)
s = elasticsearch_dsl.Search(using=es_client, index=elasticsearch_index)
s = s.params(size=0)
s.aggs.bucket('sample_ids', elasticsearch_dsl.A('terms', field='samples_num_alt_1', size=10000))
response = s.execute()
return [agg['key'] for agg in response.aggregations.sample_ids.buckets], index_metadata
def get_elasticsearch_index_samples(elasticsearch_index, dataset_type=Sample.DATASET_TYPE_VARIANT_CALLS):
es_client = get_es_client()
index_metadata = get_index_metadata(elasticsearch_index, es_client).get(elasticsearch_index)
s = elasticsearch_dsl.Search(using=es_client, index=elasticsearch_index)
s = s.params(size=0)
s.aggs.bucket('sample_ids', elasticsearch_dsl.A('terms', field=SAMPLE_FIELDS_MAP[dataset_type], size=10000))
response = s.execute()
return [agg['key'] for agg in response.aggregations.sample_ids.buckets], index_metadata
def elasticsearch_status(request):
client = get_es_client()
disk_fields = ['node', 'shards', 'disk.avail', 'disk.used', 'disk.percent']
disk_status = [{
_to_camel_case(field.replace('.', '_')): disk[field] for field in disk_fields
} for disk in client.cat.allocation(format="json", h=','.join(disk_fields))]
index_fields = ['index', 'docs.count', 'store.size', 'creation.date.string']
indices = [{
_to_camel_case(field.replace('.', '_')): index[field] for field in index_fields
} for index in client.cat.indices(format="json", h=','.join(index_fields))
if all(not index['index'].startswith(omit_prefix) for omit_prefix in ['.', 'index_operations_log'])]
aliases = defaultdict(list)
for alias in client.cat.aliases(format="json", h='alias,index'):
aliases[alias['alias']].append(alias['index'])
index_metadata = get_index_metadata('_all', client, use_cache=False)
active_samples = Sample.objects.filter(is_active=True).select_related('individual__family__project')
seqr_index_projects = defaultdict(lambda: defaultdict(set))
es_projects = set()
for sample in active_samples:
for index_name in sample.elasticsearch_index.split(','):
project = sample.individual.family.project
es_projects.add(project)
if index_name in aliases:
for aliased_index_name in aliases[index_name]:
seqr_index_projects[aliased_index_name][project].add(sample.individual.guid)
else:
seqr_index_projects[index_name.rstrip('*')][project].add(sample.individual.guid)
for index in indices:
index_name = index['index']
index.update(index_metadata[index_name])
projects_for_index = []
for index_prefix in list(seqr_index_projects.keys()):
if index_name.startswith(index_prefix):
projects_for_index += list(seqr_index_projects.pop(index_prefix).keys())
index['projects'] = [{'projectGuid': project.guid, 'projectName': project.name} for project in projects_for_index]
errors = ['{} does not exist and is used by project(s) {}'.format(
index, ', '.join(['{} ({} samples)'.format(p.name, len(indivs)) for p, indivs in project_individuals.items()])
) for index, project_individuals in seqr_index_projects.items() if project_individuals]
return create_json_response({
'indices': indices,
'diskStats': disk_status,
'elasticsearchHost': ELASTICSEARCH_SERVER,
'errors': errors,
})
def get_elasticsearch_index_samples(elasticsearch_index):
es_client = get_es_client()
index_metadata = get_index_metadata(
elasticsearch_index, es_client,
include_fields=True).get(elasticsearch_index)
sample_field = next((field for field in SAMPLE_FIELDS_LIST
if field in index_metadata['fields'].keys()))
s = elasticsearch_dsl.Search(using=es_client, index=elasticsearch_index)
s = s.params(size=0)
s.aggs.bucket('sample_ids',
elasticsearch_dsl.A('terms', field=sample_field, size=10000))
response = s.execute()
return [agg['key'] for agg in response.aggregations.sample_ids.buckets
], index_metadata
def _get_es_indices(client):
indices = _get_es_meta(
client,
'indices',
['index', 'docs.count', 'store.size', 'creation.date.string'],
filter_rows=lambda index: all(not index['index'].startswith(
omit_prefix) for omit_prefix in ['.', 'index_operations_log']))
aliases = defaultdict(list)
for alias in _get_es_meta(client, 'aliases', ['alias', 'index']):
aliases[alias['alias']].append(alias['index'])
index_metadata = get_index_metadata('_all', client, use_cache=False)
active_samples = Sample.objects.filter(
is_active=True).select_related('individual__family__project')
seqr_index_projects = defaultdict(lambda: defaultdict(set))
es_projects = set()
for sample in active_samples:
for index_name in sample.elasticsearch_index.split(','):
project = sample.individual.family.project
es_projects.add(project)
if index_name in aliases:
for aliased_index_name in aliases[index_name]:
seqr_index_projects[aliased_index_name][project].add(
sample.individual.guid)
else:
seqr_index_projects[index_name.rstrip('*')][project].add(
sample.individual.guid)
for index in indices:
index_name = index['index']
index.update(index_metadata[index_name])
projects_for_index = []
for index_prefix in list(seqr_index_projects.keys()):
if index_name.startswith(index_prefix):
projects_for_index += list(
seqr_index_projects.pop(index_prefix).keys())
index['projects'] = [{
'projectGuid': project.guid,
'projectName': project.name
} for project in projects_for_index]
return indices, seqr_index_projects