missing="NA",
quote='"',
types={
'gene_name': TString(),
'description': TString(),
'gene_id': TString(),
'xcase_lof': TInt(),
'xctrl_lof': TInt(),
'pval_lof': TDouble(),
'xcase_mpc': TInt(),
'xctrl_mpc': TInt(),
'pval_mpc': TDouble(),
'xcase_infrIndel': TInt(),
'xctrl_infrIndel': TInt(),
'pval_infrIndel': TDouble(),
'pval_meta': TDouble(),
'analysis_group': TString(),
})
es = ElasticsearchClient(args.host, args.port)
es.export_kt_to_elasticsearch(
kt,
index_name="epi_exome_gene_results_181107",
index_type_name="result",
block_size=args.block_size,
num_shards=args.num_shards,
delete_index_before_exporting=True,
verbose=True,
)
for group in analysis_groups:
group_results = variant_results.filter('analysis_group == "%s"' % group).drop("analysis_group")
group_results = group_results.annotate(
"%s = { %s }" % (group, ", ".join(["%s: %s" % (col, col) for col in result_columns]))
).select(["v", group])
variants = variants.join(group_results.key_by("v"))
variants = variants.annotate("groups = { %s }" % ", ".join(["%s:%s" % (group, group) for group in analysis_groups]))
variants = variants.drop(list(analysis_groups))
variants = variants.annotate("v = Variant(v)")
variants = variants.annotate("variant_id = %s" % get_expr_for_variant_id())
variants = variants.annotate("chrom = %s" % get_expr_for_contig())
variants = variants.annotate("pos = %s" % get_expr_for_start_pos())
variants = variants.annotate("xpos = %s" % get_expr_for_xpos())
variants = variants.drop(["v"])
pprint.pprint(variants.schema)
es = ElasticsearchClient(args.host, args.port)
es.export_kt_to_elasticsearch(
variants,
index_name=args.index,
index_type_name="variant",
block_size=args.block_size,
num_shards=args.num_shards,
delete_index_before_exporting=True,
verbose=True,
)
for field_name in transcript_annotations_to_keep:
new_field_name = field_name.split("_")[0] + "".join(
map(lambda word: word.capitalize(),
field_name.split("_")[1:]))
combined_kt = combined_kt.annotate(
"%(new_field_name)s = mainTranscript.%(field_name)s" % locals())
combined_kt = combined_kt.drop(["mainTranscript"])
pprint(combined_kt.schema)
DISABLE_INDEX_AND_DOC_VALUES_FOR_FIELDS = ("sortedTranscriptConsequences", )
print("======== Export to elasticsearch ======")
es = ElasticsearchClient(
host=args.host,
port=args.port,
)
es.export_kt_to_elasticsearch(
combined_kt,
index_name=args.index,
index_type_name=args.index_type,
block_size=args.block_size,
num_shards=args.num_shards,
delete_index_before_exporting=True,
disable_doc_values_for_fields=DISABLE_INDEX_AND_DOC_VALUES_FOR_FIELDS,
disable_index_for_fields=DISABLE_INDEX_AND_DOC_VALUES_FOR_FIELDS,
verbose=True,
)
COVERAGE_PATHS = EXOME_COVERAGE_CSV_PATHS[-1]
kt_coverage = hc.import_table(COVERAGE_PATHS, types=types)
kt_coverage = kt_coverage.rename({
'#chrom': 'chrom',
'1': 'over1',
'5': 'over5',
'10': 'over10',
'15': 'over15',
'20': 'over20',
'25': 'over25',
'30': 'over30',
'50': 'over50',
'100': 'over100',
})
print(kt_coverage.schema)
print("======== Export exome coverage to elasticsearch ======")
es = ElasticsearchClient(
host=args.host,
port=args.port,
)
es.export_kt_to_elasticsearch(kt_coverage,
index_name=args.index,
index_type_name=args.index_type,
num_shards=args.num_shards,
block_size=args.block_size,
delete_index_before_exporting=True,
verbose=True)
kt = hc.read_table(gene_results_url)
kt = kt.rename({
'ensembl_gene_id': 'gene_id',
'Xcase_lof': 'xcase_lof',
'Xctrl_lof': 'xctrl_lof',
'Pval_lof': 'pval_lof',
'Xcase_mpc': 'xcase_mpc',
'Xctrl_mpc': 'xctrl_mpc',
'Pval_mpc': 'pval_mpc',
'Pval_meta': 'pval_meta',
})
kt = kt.annotate("analysis_group = \"all\"")
es = ElasticsearchClient(
host=args.host,
port=args.port,
)
es.export_kt_to_elasticsearch(
kt,
index_name="schizophrenia_gene_results_171213",
index_type_name="result",
block_size=args.block_size,
num_shards=args.num_shards,
delete_index_before_exporting=True,
verbose=True,
)