def setUp(self): self.hsq1 = 0.2 self.hsq2 = 0.7 ld = (np.abs(np.random.normal(size=800)) + 1).reshape((400, 2)) N = np.ones((400, 1)) * 1e5 self.M = np.ones((1, 2)) * 1e7 / 2.0 chisq = 1 + 1e5 * (ld[:, 0] * self.hsq1 / self.M[0, 0] + ld[:, 1] * self.hsq2 / self.M[0, 1]).reshape((400, 1)) w_ld = np.ones_like(chisq) self.hsq_noint = reg.Hsq( chisq, ld, w_ld, N, self.M, n_blocks=3, intercept=1) self.hsq_int = reg.Hsq(chisq, ld, w_ld, N, self.M, n_blocks=3) print(self.hsq_noint.summary()) print(self.hsq_int.summary())
def test_eq_hsq(self): ''' Gencov should be the same as hsq if z1 = z2, hsq + intercept_hsq are 0 and all intermediate rg's are > 0 (because Hsq.weights lower-bounds the hsq guess at 0 but Gencov.weights lower-bounds the rho_g guess at -1). The setup below guarantees that all intermediate rho_g guesses will be 1 ''' self.ld = np.abs(np.random.normal(size=100).reshape((50, 2))) + 2 self.z1 = (np.sum(self.ld, axis=1) + 10).reshape((50, 1)) gencov = reg.Gencov(self.z1, self.z1, self.ld, self.w_ld, self.N1, self.N1, self.M, 0, 0, 0, 0, n_blocks=3, intercept_gencov=1) hsq = reg.Hsq(np.square(self.z1), self.ld, self.w_ld, self.N1, self.M, n_blocks=3, intercept=1) print(gencov.summary(['asdf', 'asdf'])) print(hsq.summary(['asdf', 'asdf'])) assert_array_almost_equal(gencov.tot, hsq.tot) assert_array_almost_equal(gencov.tot_se, hsq.tot_se) assert_array_almost_equal(gencov.tot_cov, hsq.tot_cov)
def cell_type_specific(args, log): '''Cell type specific analysis''' args = copy.deepcopy(args) if args.intercept_h2 is not None: args.intercept_h2 = float(args.intercept_h2) if args.no_intercept: args.intercept_h2 = 1 M_annot_all_regr, w_ld_cname, ref_ld_cnames_all_regr, sumstats, novar_cols = \ _read_ld_sumstats(args, log, args.h2_cts) M_tot = np.sum(M_annot_all_regr) _check_ld_condnum(args, log, ref_ld_cnames_all_regr) _warn_length(log, sumstats) n_snp = len(sumstats) n_blocks = min(n_snp, args.n_blocks) if args.chisq_max is None: chisq_max = max(0.001*sumstats.N.max(), 80) else: chisq_max = args.chisq_max ii = np.ravel(sumstats.Z**2 < chisq_max) sumstats = sumstats.iloc[ii, :] log.log('Removed {M} SNPs with chi^2 > {C} ({N} SNPs remain)'.format( C=chisq_max, N=np.sum(ii), M=n_snp-np.sum(ii))) n_snp = np.sum(ii) # lambdas are late-binding, so this works ref_ld_all_regr = np.array(sumstats[ref_ld_cnames_all_regr]).reshape((len(sumstats),-1)) chisq = np.array(sumstats.Z**2) keep_snps = sumstats[['SNP']] s = lambda x: np.array(x).reshape((n_snp, 1)) results_columns = ['Name', 'Coefficient', 'Coefficient_std_error', 'Coefficient_P_value'] results_data = [] for (name, ct_ld_chr) in [x.split() for x in open(args.ref_ld_chr_cts).readlines()]: ref_ld_cts_allsnps = _read_chr_split_files(ct_ld_chr, None, log, 'cts reference panel LD Score', ps.ldscore_fromlist) log.log('Performing regression.') ref_ld_cts = np.array(pd.merge(keep_snps, ref_ld_cts_allsnps, on='SNP', how='left').iloc[:,1:]) if np.any(np.isnan(ref_ld_cts)): raise ValueError ('Missing some LD scores from cts files. Are you sure all SNPs in ref-ld-chr are also in ref-ld-chr-cts') ref_ld = np.hstack([ref_ld_cts, ref_ld_all_regr]) M_cts = ps.M_fromlist( _splitp(ct_ld_chr), _N_CHR, common=(not args.not_M_5_50)) M_annot = np.hstack([M_cts, M_annot_all_regr]) hsqhat = reg.Hsq(s(chisq), ref_ld, s(sumstats[w_ld_cname]), s(sumstats.N), M_annot, n_blocks=n_blocks, intercept=args.intercept_h2, twostep=None, old_weights=True) coef, coef_se = hsqhat.coef[0], hsqhat.coef_se[0] results_data.append((name, coef, coef_se, stats.norm.sf(coef/coef_se))) if args.print_all_cts: for i in range(1, len(ct_ld_chr.split(','))): coef, coef_se = hsqhat.coef[i], hsqhat.coef_se[i] results_data.append((name+'_'+str(i), coef, coef_se, stats.norm.sf(coef/coef_se))) df_results = pd.DataFrame(data = results_data, columns = results_columns) df_results.sort_values(by = 'Coefficient_P_value', inplace=True) df_results.to_csv(args.out+'.cell_type_results.txt', sep='\t', index=False) log.log('Results printed to '+args.out+'.cell_type_results.txt')
def setUp(self): self.chisq = np.ones((4, 1)) * 4 self.ld = np.ones((4, 1)) self.w_ld = np.ones((4, 1)) self.N = 9 * np.ones((4, 1)) self.M = np.matrix((7)) self.hsq = reg.Hsq( self.chisq, self.ld, self.w_ld, self.N, self.M, n_blocks=3, intercept=1)
def setUp(self): self.chisq = np.ones((17, 1)) * 4 self.ld = np.hstack( [np.ones((17, 1)), np.arange(17).reshape((17, 1))]).reshape((17, 2)) self.w_ld = np.ones((17, 1)) self.N = 9 * np.ones((17, 1)) self.M = np.matrix((7, 2)) self.hsq = reg.Hsq( self.chisq, self.ld, self.w_ld, self.N, self.M, n_blocks=3, intercept=1)
def test_summary(self): # not much to test; we can at least make sure no errors at runtime self.hsq.summary(['asdf']) self.ld += np.arange(4).reshape((4, 1)) self.chisq += np.arange(4).reshape((4, 1)) hsq = reg.Hsq( self.chisq, self.ld, self.w_ld, self.N, self.M, n_blocks=3) hsq.summary(['asdf']) # test ratio printout with mean chi^2 < 1 hsq.mean_chisq = 0.5 hsq.summary(['asdf'])
def test_summary(self): # not much to test; we can at least make sure no errors at runtime self.hsq.summary(['asdf', 'qwer']) # change to random 7/30/2019 to avoid inconsistent singular matrix errors self.ld += np.random.normal(scale=0.1, size=(17, 2)) self.chisq += np.arange(17).reshape((17, 1)) hsq = reg.Hsq( self.chisq, self.ld, self.w_ld, self.N, self.M, n_blocks=3) hsq.summary(['asdf', 'qwer']) # test ratio printout with mean chi^2 < 1 hsq.mean_chisq = 0.5 hsq.summary(['asdf', 'qwer'])
def estimate_h2(args, log): '''Estimate h2 and partitioned h2.''' args = copy.deepcopy(args) if args.samp_prev is not None and args.pop_prev is not None: args.samp_prev, args.pop_prev = map( float, [args.samp_prev, args.pop_prev]) if args.intercept_h2 is not None: args.intercept_h2 = float(args.intercept_h2) if args.no_intercept: args.intercept_h2 = 1 M_annot, w_ld_cname, ref_ld_cnames, sumstats, novar_cols = _read_ld_sumstats( args, log, args.h2) ref_ld = np.array(sumstats[ref_ld_cnames]) _check_ld_condnum(args, log, ref_ld_cnames) _warn_length(log, sumstats) n_snp = len(sumstats) n_blocks = min(n_snp, args.n_blocks) n_annot = len(ref_ld_cnames) chisq_max = args.chisq_max old_weights = False if n_annot == 1: if args.two_step is None and args.intercept_h2 is None: args.two_step = 30 else: old_weights = True if args.chisq_max is None: chisq_max = max(0.001*sumstats.N.max(), 80) s = lambda x: np.array(x).reshape((n_snp, 1)) chisq = s(sumstats.Z**2) if chisq_max is not None: ii = np.ravel(chisq < chisq_max) sumstats = sumstats.iloc[ii, :] log.log('Removed {M} SNPs with chi^2 > {C} ({N} SNPs remain)'.format( C=chisq_max, N=np.sum(ii), M=n_snp-np.sum(ii))) n_snp = np.sum(ii) # lambdas are late-binding, so this works ref_ld = np.array(sumstats[ref_ld_cnames]) chisq = chisq[ii].reshape((n_snp, 1)) if args.two_step is not None: log.log('Using two-step estimator with cutoff at {M}.'.format(M=args.two_step)) hsqhat = reg.Hsq(chisq, ref_ld, s(sumstats[w_ld_cname]), s(sumstats.N), M_annot, n_blocks=n_blocks, intercept=args.intercept_h2, twostep=args.two_step, old_weights=old_weights) if args.print_cov: _print_cov(hsqhat, args.out + '.cov', log) if args.print_delete_vals: _print_delete_values(hsqhat, args.out + '.delete', log) _print_part_delete_values(hsqhat, args.out + '.part_delete', log) log.log(hsqhat.summary(ref_ld_cnames, P=args.samp_prev, K=args.pop_prev, overlap = args.overlap_annot)) if args.overlap_annot: overlap_matrix, M_tot = _read_annot(args, log) # overlap_matrix = overlap_matrix[np.array(~novar_cols), np.array(~novar_cols)]#np.logical_not df_results = hsqhat._overlap_output(ref_ld_cnames, overlap_matrix, M_annot, M_tot, args.print_coefficients) df_results.to_csv(args.out+'.results', sep="\t", index=False) log.log('Results printed to '+args.out+'.results') return hsqhat