def export_loo(batch_size=256, update=False):
r'''
For exporting p-values of meta-analysis of leave-one-out population sets
'''
meta_mt0 = hl.read_matrix_table(get_meta_analysis_results_path())
meta_mt0 = meta_mt0.select_rows()
meta_mt0 = meta_mt0.annotate_cols(pheno_id = get_pheno_id(tb=meta_mt0))
meta_mt0 = meta_mt0.filter_cols(hl.len(meta_mt0.pheno_data.pop)==6)
if update:
current_dir = f'{ldprune_dir}/loo/sumstats/batch1' # directory of current results to update
ss_list = hl.hadoop_ls(current_dir)
pheno_id_list = [x['path'].replace('.tsv.bgz','').replace(f'{current_dir}/','') for x in ss_list if 'bgz' in x['path']]
meta_mt0 = meta_mt0.filter_cols(~hl.literal(pheno_id_list).contains(meta_mt0.pheno_id))
meta_mt0 = meta_mt0.annotate_rows(chr = meta_mt0.locus.contig,
pos = meta_mt0.locus.position,
SNP = (meta_mt0.locus.contig+':'+
hl.str(meta_mt0.locus.position)+':'+
meta_mt0.alleles[0]+':'+
meta_mt0.alleles[1]))
all_pops = sorted(['AFR', 'AMR', 'CSA', 'EAS', 'EUR', 'MID'])
annotate_dict = {}
'''
pop_idx corresponds to the alphabetic ordering of the pops
entry with idx=0 is 6-pop meta-analysis, entry with idx=1 is 5-pop not-AFR
meta-analysis, idx=2 is 5-pop not-AMR, etc)
'''
for pop_idx, pop in enumerate(all_pops,1):
annotate_dict.update({f'pval_not_{pop}': meta_mt0.meta_analysis.Pvalue[pop_idx]})
meta_mt1 = meta_mt0.annotate_entries(**annotate_dict)
meta_mt1 = meta_mt1.key_cols_by('pheno_id')
meta_mt1 = meta_mt1.key_rows_by().drop('locus','alleles','meta_analysis')
meta_mt1.describe()
batch_idx = 1
get_export_path = lambda batch_idx: f'{ldprune_dir}/loo/sumstats/batch{batch_idx}'
while hl.hadoop_is_dir(get_export_path(batch_idx)):
batch_idx += 1
print(f'\nExporting to: {get_export_path(batch_idx)}\n')
print(meta_mt1.count_cols())
hl.experimental.export_entries_by_col(mt = meta_mt1,
path = get_export_path(batch_idx),
bgzip = True,
batch_size = batch_size,
use_string_key_as_file_name = True,
header_json_in_file = False)
def export_loo(batch_size=256):
r'''
For exporting p-values of meta-analysis of leave-one-out population sets
'''
meta_mt0 = hl.read_matrix_table(get_meta_analysis_results_path())
meta_mt0 = meta_mt0.filter_cols(hl.len(meta_mt0.pheno_data.pop) == 6)
meta_mt0 = meta_mt0.annotate_cols(pheno_id=(
meta_mt0.trait_type + '-' + meta_mt0.phenocode + '-' +
meta_mt0.pheno_sex +
hl.if_else(hl.len(meta_mt0.coding) > 0, '-' + meta_mt0.coding, '') +
hl.if_else(hl.len(meta_mt0.modifier) > 0, '-' +
meta_mt0.modifier, '')).replace(' ', '_').replace('/', '_'))
meta_mt0 = meta_mt0.annotate_rows(
SNP=(meta_mt0.locus.contig + ':' + hl.str(meta_mt0.locus.position) +
':' + meta_mt0.alleles[0] + ':' + meta_mt0.alleles[1]))
all_pops = sorted(['AFR', 'AMR', 'CSA', 'EAS', 'EUR', 'MID'])
annotate_dict = {}
for pop_idx, pop in enumerate(
all_pops, 1
): # pop idx corresponds to the alphabetic ordering of the pops (entry with idx=0 is 6-pop meta-analysis)
annotate_dict.update(
{f'pval_not_{pop}': meta_mt0.meta_analysis.Pvalue[pop_idx]})
meta_mt1 = meta_mt0.annotate_entries(**annotate_dict)
meta_mt1 = meta_mt1.key_cols_by('pheno_id')
meta_mt1 = meta_mt1.key_rows_by().drop('locus', 'alleles', 'gene',
'annotation', 'meta_analysis')
print(meta_mt1.describe())
batch_idx = 1
get_export_path = lambda batch_idx: f'{ldprune_dir}/loo/sumstats/batch{batch_idx}'
while hl.hadoop_is_dir(get_export_path(batch_idx)):
batch_idx += 1
print(f'\nExporting to: {get_export_path(batch_idx)}\n')
hl.experimental.export_entries_by_col(mt=meta_mt1,
path=get_export_path(batch_idx),
bgzip=True,
batch_size=batch_size,
use_string_key_as_file_name=True,
header_json_in_file=False)
def export_results(num_pops, trait_types='all', batch_size=256, mt=None,
export_path_str=None, skip_binary_eur=True):
r'''
`num_pops`: exact number of populations for which phenotype is defined
`trait_types`: trait category (options: all, binary, quant)
`batch_size`: batch size argument for export entries by col
'''
assert trait_types in {'all','quant','binary'}, "trait_types must be one of the following: {'all','quant','binary'}"
print(f'\n\nExporting {trait_types} trait types for {num_pops} pops\n\n')
if mt == None:
mt0 = get_final_sumstats_mt_for_export()
else:
mt0 = mt
meta_mt0 = hl.read_matrix_table(get_meta_analysis_results_path())
mt0 = mt0.annotate_cols(pheno_id = get_pheno_id(tb=mt0))
mt0 = mt0.annotate_rows(chr = mt0.locus.contig,
pos = mt0.locus.position,
ref = mt0.alleles[0],
alt = mt0.alleles[1])
all_pops = ['AFR', 'AMR', 'CSA', 'EAS', 'EUR', 'MID']
if trait_types == 'all':
trait_types_to_run = ['continuous','biomarkers','categorical','phecode', 'icd10', 'prescriptions'] # list of which trait_type to run
elif trait_types == 'quant':
trait_types_to_run = ['continuous','biomarkers']
elif trait_types == 'binary':
trait_types_to_run = ['categorical','phecode', 'icd10', 'prescriptions']
pop_sets = [set(i) for i in list(combinations(all_pops, num_pops))] # list of exact set of pops for which phenotype is defined
# fields specific to each category of trait
quant_meta_fields = ['AF_Allele2']
quant_fields = ['AF_Allele2']
binary_meta_fields = ['AF_Cases','AF_Controls']
binary_fields = ['AF.Cases','AF.Controls']
# dictionaries for renaming fields
quant_meta_field_rename_dict = {'AF_Allele2':'af_meta',
'BETA':'beta_meta',
'SE':'se_meta',
'Pvalue':'pval_meta',
'Pvalue_het':'pval_heterogeneity'}
quant_field_rename_dict = {'AF_Allele2':'af',
'BETA':'beta',
'SE':'se',
'Pvalue':'pval',
'low_confidence':'low_confidence'} # decided on this implementation to make later code cleaner
binary_meta_field_rename_dict = {'BETA':'beta_meta',
'SE':'se_meta',
'Pvalue':'pval_meta',
'AF_Cases':'af_cases_meta',
'AF_Controls':'af_controls_meta',
'Pvalue_het':'pval_heterogeneity'}
binary_field_rename_dict = {'AF.Cases':'af_cases',
'AF.Controls':'af_controls',
'BETA':'beta',
'SE':'se',
'Pvalue':'pval',
'low_confidence':'low_confidence'} # decided on this implementation to make later code cleaner
all_quant_trait_types = {'continuous','biomarkers'}
all_binary_trait_types = {'categorical','phecode', 'icd10', 'prescriptions'}
quant_trait_types = all_quant_trait_types.intersection(trait_types_to_run) # get list of quant trait types to run
binary_trait_types = all_binary_trait_types.intersection(trait_types_to_run) # get list of binary trait types to run
error_trait_types = set(trait_types_to_run).difference(quant_trait_types.union(binary_trait_types))
assert len(error_trait_types)==0, f'ERROR: The following trait_types are invalid: {error_trait_types}'
for trait_category, trait_types in [('binary', binary_trait_types), ('quant', quant_trait_types)]:
if len(trait_types)==0: #if no traits in trait_types list
continue
print(f'{trait_category} trait types to run: {trait_types}')
if trait_category == 'quant':
meta_fields = quant_meta_fields
fields = quant_fields
meta_field_rename_dict = quant_meta_field_rename_dict
field_rename_dict = quant_field_rename_dict
elif trait_category == 'binary':
meta_fields = binary_meta_fields
fields = binary_fields
meta_field_rename_dict = binary_meta_field_rename_dict
field_rename_dict = binary_field_rename_dict
meta_fields += ['BETA','SE','Pvalue','Pvalue_het']
fields += ['BETA','SE','Pvalue','low_confidence']
for pop_set in pop_sets:
start = time()
if (pop_set == {'EUR'} and trait_category == 'binary') and skip_binary_eur: # run EUR-only binary traits separately
print('\nSkipping EUR-only binary traits\n')
continue
mt1 = mt0.filter_cols((hl.literal(trait_types).contains(mt0.trait_type))&
(hl.set(mt0.pheno_data.pop)==hl.literal(pop_set)))
col_ct = mt1.count_cols()
if col_ct==0:
print(f'\nSkipping {trait_types},{sorted(pop_set)}, no phenotypes found\n')
continue
pop_list = sorted(pop_set)
annotate_dict = {}
keyed_mt = meta_mt0[mt1.row_key,mt1.col_key]
if len(pop_set)>1:
for field in meta_fields: # NOTE: Meta-analysis columns go before per-population columns
field_expr = keyed_mt.meta_analysis[field][0]
annotate_dict.update({f'{meta_field_rename_dict[field]}': hl.if_else(hl.is_nan(field_expr),
hl.str(field_expr),
hl.format('%.3e', field_expr))})
for field in fields:
for pop_idx, pop in enumerate(pop_list):
field_expr = mt1.summary_stats[field][pop_idx]
annotate_dict.update({f'{field_rename_dict[field]}_{pop}': hl.if_else(hl.is_nan(field_expr),
hl.str(field_expr),
hl.str(field_expr) if field=='low_confidence' else hl.format('%.3e', field_expr))})
mt2 = mt1.annotate_entries(**annotate_dict)
mt2 = mt2.filter_cols(mt2.coding != 'zekavat_20200409')
mt2 = mt2.key_cols_by('pheno_id')
mt2 = mt2.key_rows_by().drop('locus','alleles','summary_stats') # row fields that are no longer included: 'gene','annotation'
batch_idx = 1
get_export_path = lambda batch_idx: f'{ldprune_dir}/export_results/{"" if export_path_str is None else f"{export_path_str}/"}{trait_category}/{"-".join(pop_list)}_batch{batch_idx}'
print(mt2.describe())
while hl.hadoop_is_dir(get_export_path(batch_idx)):
batch_idx += 1
print(f'\nExporting {col_ct} phenos to: {get_export_path(batch_idx)}\n')
hl.experimental.export_entries_by_col(mt = mt2,
path = get_export_path(batch_idx),
bgzip = True,
batch_size = batch_size,
use_string_key_as_file_name = True,
header_json_in_file = False)
end = time()
print(f'\nExport complete for:\n{trait_types}\n{pop_list}\ntime: {round((end-start)/3600,2)} hrs')
def export_binary_eur(cluster_idx, num_clusters=10, batch_size = 256):
r'''
Export summary statistics for binary traits defined only for EUR.
Given the large number of such traits (4184), it makes sense to batch this
across `num_clusters` clusters for reduced wall time and robustness to mid-export errors.
NOTE: `cluster_idx` is 1-indexed.
'''
mt0 = get_final_sumstats_mt_for_export()
meta_mt0 = hl.read_matrix_table(get_meta_analysis_results_path())
mt0 = mt0.annotate_cols(pheno_id = get_pheno_id(tb=mt0))
mt0 = mt0.annotate_rows(chr = mt0.locus.contig,
pos = mt0.locus.position,
ref = mt0.alleles[0],
alt = mt0.alleles[1])
trait_types_to_run = ['categorical','phecode', 'icd10', 'prescriptions'] # list of which trait_type to run
# fields specific to each category of trait
meta_fields = ['AF_Cases','AF_Controls']
fields = ['AF.Cases','AF.Controls']
# dictionaries for renaming fields
meta_field_rename_dict = {'BETA':'beta_meta',
'SE':'se_meta',
'Pvalue':'pval_meta',
'AF_Cases':'af_cases_meta',
'AF_Controls':'af_controls_meta',
'Pvalue_het':'pval_heterogeneity'}
field_rename_dict = {'AF.Cases':'af_cases',
'AF.Controls':'af_controls',
'BETA':'beta',
'SE':'se',
'Pvalue':'pval',
'low_confidence':'low_confidence'} # decided on this implementation to make later code cleaner
all_binary_trait_types = {'categorical','phecode', 'icd10', 'prescriptions'}
meta_fields += ['BETA','SE','Pvalue','Pvalue_het']
fields += ['BETA','SE','Pvalue','low_confidence']
trait_category = 'binary'
trait_types = all_binary_trait_types.intersection(trait_types_to_run) # get list of binary trait types to run
pop_set = {'EUR'}
start = time()
mt1 = mt0.filter_cols((hl.literal(trait_types).contains(mt0.trait_type))&
(hl.set(mt0.pheno_data.pop)==hl.literal(pop_set)))
pheno_id_list = mt1.pheno_id.collect()
num_traits = len(pheno_id_list) # total number of traits to run
traits_per_cluster = ceil(num_traits/num_clusters) # maximum traits to run per cluster
cluster_pheno_id_list = pheno_id_list[(cluster_idx-1)*traits_per_cluster:cluster_idx*traits_per_cluster] # list of traits to run in current cluster
print(len(cluster_pheno_id_list))
mt1 = mt1.filter_cols(hl.literal(cluster_pheno_id_list).contains(mt1.pheno_id))
pop_list = sorted(pop_set)
annotate_dict = {}
keyed_mt = meta_mt0[mt1.row_key,mt1.col_key]
if len(pop_set)>1:
for field in meta_fields: # NOTE: Meta-analysis columns go before per-population columns
field_expr = keyed_mt.meta_analysis[field][0]
annotate_dict.update({f'{meta_field_rename_dict[field]}': hl.if_else(hl.is_nan(field_expr),
hl.str(field_expr),
hl.format('%.3e', field_expr))})
for field in fields:
for pop_idx, pop in enumerate(pop_list):
field_expr = mt1.summary_stats[field][pop_idx]
annotate_dict.update({f'{field_rename_dict[field]}_{pop}': hl.if_else(hl.is_nan(field_expr),
hl.str(field_expr),
hl.str(field_expr) if field=='low_confidence' else hl.format('%.3e', field_expr))})
mt2 = mt1.annotate_entries(**annotate_dict)
mt2 = mt2.filter_cols(mt2.coding != 'zekavat_20200409')
mt2 = mt2.key_cols_by('pheno_id')
mt2 = mt2.key_rows_by().drop('locus','alleles','summary_stats') # row fields that are no longer included: 'gene','annotation'
print(mt2.describe())
batch_idx = 1
get_export_path = lambda batch_idx: f'{ldprune_dir}/release/{trait_category}/{"-".join(pop_list)}_batch{batch_idx}/subbatch{cluster_idx}'
while hl.hadoop_is_dir(get_export_path(batch_idx)):
batch_idx += 1
print(f'\nExporting {len(cluster_pheno_id_list)} phenos to: {get_export_path(batch_idx)}\n')
hl.experimental.export_entries_by_col(mt = mt2,
path = get_export_path(batch_idx),
bgzip = True,
batch_size = batch_size,
use_string_key_as_file_name = True,
header_json_in_file = False)
end = time()
print(f'\nExport complete for:\n{trait_types}\n{pop_list}\ntime: {round((end-start)/3600,2)} hrs')