def _read_M(args, log, n_annot): '''Read M (--M, --M-file, etc).''' if args.M: try: M_annot = [float(x) for x in _splitp(args.M)] except ValueError as e: raise ValueError('Could not cast --M to float: ' + str(e.args)) else: if args.ref_ld: M_annot = ps.M_fromlist(_splitp(args.ref_ld), common=(not args.not_M_5_50)) elif args.ref_ld_chr: M_annot = ps.M_fromlist(_splitp(args.ref_ld_chr), _N_CHR, common=(not args.not_M_5_50)) try: M_annot = np.array(M_annot).reshape((1, n_annot)) except ValueError as e: print M_annot print n_annot raise ValueError( '# terms in --M must match # of LD Scores in --ref-ld.\n' + str(e.args)) return M_annot
def cell_type_specific(args, log): '''Cell type specific analysis''' args = copy.deepcopy(args) if args.intercept_h2 is not None: args.intercept_h2 = float(args.intercept_h2) if args.no_intercept: args.intercept_h2 = 1 M_annot_all_regr, w_ld_cname, ref_ld_cnames_all_regr, sumstats, novar_cols = \ _read_ld_sumstats(args, log, args.h2_cts) M_tot = np.sum(M_annot_all_regr) _check_ld_condnum(args, log, ref_ld_cnames_all_regr) _warn_length(log, sumstats) n_snp = len(sumstats) n_blocks = min(n_snp, args.n_blocks) if args.chisq_max is None: chisq_max = max(0.001*sumstats.N.max(), 80) else: chisq_max = args.chisq_max ii = np.ravel(sumstats.Z**2 < chisq_max) sumstats = sumstats.ix[ii, :] log.log('Removed {M} SNPs with chi^2 > {C} ({N} SNPs remain)'.format( C=chisq_max, N=np.sum(ii), M=n_snp-np.sum(ii))) n_snp = np.sum(ii) # lambdas are late-binding, so this works ref_ld_all_regr = np.array(sumstats[ref_ld_cnames_all_regr]).reshape((len(sumstats),-1)) chisq = np.array(sumstats.Z**2) keep_snps = sumstats[['SNP']] s = lambda x: np.array(x).reshape((n_snp, 1)) results_columns = ['Name', 'Coefficient', 'Coefficient_std_error', 'Coefficient_P_value'] results_data = [] for (name, ct_ld_chr) in [x.split() for x in open(args.ref_ld_chr_cts).readlines()]: ref_ld_cts_allsnps = _read_chr_split_files(ct_ld_chr, None, log, 'cts reference panel LD Score', ps.ldscore_fromlist) log.log('Performing regression.') ref_ld_cts = np.array(pd.merge(keep_snps, ref_ld_cts_allsnps, on='SNP', how='left').ix[:,1:]) if np.any(np.isnan(ref_ld_cts)): raise ValueError ('Missing some LD scores from cts files. Are you sure all SNPs in ref-ld-chr are also in ref-ld-chr-cts') ref_ld = np.hstack([ref_ld_cts, ref_ld_all_regr]) M_cts = ps.M_fromlist( _splitp(ct_ld_chr), _N_CHR, common=(not args.not_M_5_50)) M_annot = np.hstack([M_cts, M_annot_all_regr]) hsqhat = reg.Hsq(s(chisq), ref_ld, s(sumstats[w_ld_cname]), s(sumstats.N), M_annot, n_blocks=n_blocks, intercept=args.intercept_h2, twostep=None, old_weights=True) coef, coef_se = hsqhat.coef[0], hsqhat.coef_se[0] results_data.append((name, coef, coef_se, stats.norm.sf(coef/coef_se))) if args.print_all_cts: for i in range(1, len(ct_ld_chr.split(','))): coef, coef_se = hsqhat.coef[i], hsqhat.coef_se[i] results_data.append((name+'_'+str(i), coef, coef_se, stats.norm.sf(coef/coef_se))) df_results = pd.DataFrame(data = results_data, columns = results_columns) df_results.sort_values(by = 'Coefficient_P_value', inplace=True) df_results.to_csv(args.out+'.cell_type_results.txt', sep='\t', index=False) log.log('Results printed to '+args.out+'.cell_type_results.txt')
def _read_M(args, n_annot, ref_indices): '''Read M (--M, --M-file, etc).''' if args.ref_ld: M_annot = ps.M_fromlist(_splitp(args.ref_ld), ref_indices, common=True) elif args.ref_ld_chr: M_annot = ps.M_fromlist(_splitp(args.ref_ld_chr), ref_indices, num=_N_CHR, common=True) try: M_annot = np.array(M_annot).reshape((1, n_annot)) except ValueError as e: raise ValueError( '# terms in --M must match # of LD Scores in --ref-ld.\n' + str(e.args)) return M_annot
def cell_type_specific(args, log): '''Cell type specific analysis''' args = copy.deepcopy(args) if args.intercept_h2 is not None: args.intercept_h2 = float(args.intercept_h2) if args.no_intercept: args.intercept_h2 = 1 M_annot_all_regr, w_ld_cname, ref_ld_cnames_all_regr, sumstats, novar_cols = \ _read_ld_sumstats(args, log, args.h2_cts) M_tot = np.sum(M_annot_all_regr) _check_ld_condnum(args, log, ref_ld_cnames_all_regr) _warn_length(log, sumstats) n_snp = len(sumstats) n_blocks = min(n_snp, args.n_blocks) if args.chisq_max is None: chisq_max = max(0.001 * sumstats.N.max(), 80) else: chisq_max = args.chisq_max ii = np.ravel(sumstats.Z**2 < chisq_max) sumstats = sumstats.ix[ii, :] log.log('Removed {M} SNPs with chi^2 > {C} ({N} SNPs remain)'.format( C=chisq_max, N=np.sum(ii), M=n_snp - np.sum(ii))) n_snp = np.sum(ii) # lambdas are late-binding, so this works ref_ld_all_regr = np.array(sumstats[ref_ld_cnames_all_regr]).reshape( (len(sumstats), -1)) chisq = np.array(sumstats.Z**2) keep_snps = sumstats[['SNP']] s = lambda x: np.array(x).reshape((n_snp, 1)) results_columns = [ 'Name', 'Coefficient', 'Coefficient_std_error', 'Coefficient_P_value' ] results_data = [] # for (name, ct_ld_chr) in [x.split() for x in open(args.ref_ld_chr_cts).readlines()]: # ORIG cts_lines = open(args.ref_ld_chr_cts).readlines() for cts_linenum, cts_line in enumerate(cts_lines, start=1): try: (name, ct_ld_chr) = cts_line.split( ) # whitespace delim file with ONLY two cols. Statement raises exception 'ValueError: too many values to unpack (expected 2)' if .split() gives more string splits. ref_ld_cts_allsnps = _read_chr_split_files( ct_ld_chr, None, log, 'cts reference panel LD Score', ps.ldscore_fromlist) log.log('Performing regression #{}/#{}. CTS name is {}'.format( cts_linenum, len(cts_lines), name)) # PT MODIFIED. sys.stdout.flush() # PT ADDED ref_ld_cts = np.array( pd.merge(keep_snps, ref_ld_cts_allsnps, on='SNP', how='left').ix[:, 1:]) if np.any(np.isnan(ref_ld_cts)): raise ValueError( 'Missing some LD scores from cts files. Are you sure all SNPs in ref-ld-chr are also in ref-ld-chr-cts' ) ref_ld = np.hstack([ref_ld_cts, ref_ld_all_regr]) M_cts = ps.M_fromlist(_splitp(ct_ld_chr), _N_CHR, common=(not args.not_M_5_50)) M_annot = np.hstack([M_cts, M_annot_all_regr]) hsqhat = reg.Hsq(s(chisq), ref_ld, s(sumstats[w_ld_cname]), s(sumstats.N), M_annot, n_blocks=n_blocks, intercept=args.intercept_h2, twostep=None, old_weights=True) coef, coef_se = hsqhat.coef[0], hsqhat.coef_se[0] results_data.append( (name, coef, coef_se, stats.norm.sf(coef / coef_se))) df_results_tmp = pd.DataFrame(data=results_data, columns=results_columns) # PT ADD df_results_tmp.to_csv(args.out + '.cell_type_results.tmp.txt', sep='\t', index=False) # PT ADD if args.print_all_cts: for i in range(1, len(ct_ld_chr.split(','))): coef, coef_se = hsqhat.coef[i], hsqhat.coef_se[i] results_data.append((name + '_' + str(i), coef, coef_se, stats.norm.sf(coef / coef_se))) except Exception as e: # e.g may catch numpy.linalg.linalg.LinAlgError: Singular matrix. log.log( '*CTS ERROR* Caught exception during regression #{}/#{}. CTS name is {}. Exception:\n{}' .format(cts_linenum, len(cts_lines), name, e)) # PT MODIFIED. sys.stdout.flush() # PT ADDED df_results = pd.DataFrame(data=results_data, columns=results_columns) df_results.sort_values(by='Coefficient_P_value', inplace=True) df_results.to_csv(args.out + '.cell_type_results.txt', sep='\t', index=False) log.log('Results printed to ' + args.out + '.cell_type_results.txt')