def run(args): start = timer() if os.path.exists(args.output): logging.info( "%s already exists, you have to move it or delete it if you want it done again", args.output) return logging.info("Creating context") context = CrossModelUtilities.context_from_args(args) results = [] n_genes = context.get_n_genes() reporter = Utilities.PercentReporter(logging.INFO, n_genes) logging.info("Processing") reporter.update(0, "%d %% of model's genes processed so far") for i, gene in enumerate(context.get_genes()): logging.log(7, "Gene %d/%d: %s", i + 1, n_genes, gene) result = JointAnalysis.joint_analysis(context, gene) results.append(result) reporter.update(i, "%d %% of model's genes processed so far") results = JointAnalysis.format_results(results) Utilities.ensure_requisite_folders(args.output) results.to_csv(args.output, index=False, sep="\t") end = timer() logging.info("Ran multi tissue in %s seconds" % (str(end - start)))
def run(args): start = timer() if os.path.exists(args.output): logging.info("%s already exists, you have to move it or delete it if you want it done again", args.output) return if (args.hdf5_expression_file and args.expression_file) or \ (not args.hdf5_expression_file and not args.expression_file): logging.info("Provide either hdf5 expression file or plain text expression file") return with PrediXcanUtilities.p_context_from_args(args) as context: genes = context.get_genes() n_genes = len(genes) reporter = Utilities.PercentReporter(logging.INFO, n_genes) reporter.update(0, "%d %% of model's genes processed so far", force=True) results = [] for i,gene in enumerate(genes): logging.log(7, "Processing gene %s", gene) r = PrediXcanAssociation.predixcan_association(gene, context) results.append(r) reporter.update(i, "%d %% of model's genes processed so far") reporter.update(i, "%d %% of model's genes processed so far") results = PrediXcanAssociation.dataframe_from_results(results) results = results.fillna("NA") results = results.sort_values(by="pvalue") Utilities.save_dataframe(results, args.output) end = timer() logging.info("Ran multi tissue predixcan in %s seconds" % (str(end - start)))
def run(args, _gwas=None): start = timer() if not args.overwrite and os.path.exists(args.output_file): logging.info("%s already exists, move it or delete it if you want it done again", args.output_file) return logging.info("Started metaxcan association") context = MetaxcanUtilities.build_context(args, _gwas) model_snps = context.get_model_snps() total_snps = len(model_snps) snps_found=set() reporter = Utilities.PercentReporter(logging.INFO, total_snps) i_genes, i_snps = context.get_data_intersection() results = [] for gene in i_genes: r, snps = AssociationCalculation.association(gene, context, return_snps=True) results.append(r) snps_found.update(snps) reporter.update(len(snps_found), "%d %% of model's snps found so far in the gwas study") Utilities.ensure_requisite_folders(args.output_file) reporter.update(len(snps_found), "%d %% of model's snps used", force=True) results = AssociationCalculation.dataframe_from_results(zip(*results)) results = MetaxcanUtilities.format_output(results, context, args.keep_ens_version) results.to_csv(args.output_file, index=False) end = timer() logging.info("Sucessfully processed metaxcan association in %s seconds"%(str(end - start)))
def run_metaxcan(args, context): logging.info("Started metaxcan association") model_snps = context.get_model_snps() total_snps = len(model_snps) snps_found=set() reporter = Utilities.PercentReporter(logging.INFO, total_snps) i_genes, i_snps = context.get_data_intersection() results = [] for gene in i_genes: logging.log(7, "Processing gene %s", gene) r, snps = AssociationCalculation.association(gene, context, return_snps=True) results.append(r) snps_found.update(snps) reporter.update(len(snps_found), "%d %% of model's snps found so far in the gwas study") reporter.update(len(snps_found), "%d %% of model's snps used", force=True) results = AssociationCalculation.dataframe_from_results(results) results = MetaxcanUtilities.format_output(results, context, args.remove_ens_version) if args.output_file: Utilities.ensure_requisite_folders(args.output_file) results.to_csv(args.output_file, index=False) return results
def run(args): start = timer() if os.path.exists(args.output): logging.info( "%s already exists, you have to move it or delete it if you want it done again", args.output) return if (args.hdf5_expression_folder and args.expression_folder) or \ (not args.hdf5_expression_folder and not args.expression_folder): logging.info( "Provide either hdf5 expression folder or plain text expression folder" ) return with MultiPrediXcanUtilities.mp_context_from_args(args) as context: genes = context.get_genes() n_genes = len(genes) reporter = Utilities.PercentReporter(logging.INFO, n_genes) reporter.update(0, "%d %% of model's genes processed so far", force=True) results = [] callbacks = {} if args.coefficient_output: callbacks["coefficient"] = MultiPrediXcanAssociation.SaveCoefs() if args.loadings_output: callbacks["loadings"] = MultiPrediXcanAssociation.SaveLoadings() for i, gene in enumerate(genes): logging.log(7, "Processing gene %i/%i: %s", i + 1, n_genes, gene) r = MultiPrediXcanAssociation.multi_predixcan_association( gene, context, callbacks.values()) results.append(r) reporter.update(i, "%d %% of model's genes processed so far") reporter.update(i, "%d %% of model's genes processed so far") results = MultiPrediXcanAssociation.dataframe_from_results( results, context) results = results.fillna("NA") results = results.sort_values(by="pvalue") Utilities.save_dataframe(results, args.output) if args.coefficient_output: Utilities.save_dataframe(callbacks["coefficient"].get(), args.coefficient_output) if args.loadings_output: Utilities.save_dataframe(callbacks["loadings"].get(), args.loadings_output) end = timer() logging.info("Ran multi tissue predixcan in %s seconds" % (str(end - start)))
def run_metaxcan(args, context): logging.info("Started metaxcan association") model_snps = context.get_model_snps() total_snps = len(model_snps) snps_found = set() reporter = Utilities.PercentReporter(logging.INFO, total_snps) i_genes, i_snps = context.get_data_intersection() results = [] additional = [] for i, gene in enumerate(i_genes): if args.MAX_R and i + 1 > args.MAX_R: logging.log("Early exit condition met") break logging.log(9, "Processing gene %i:%s", i, gene) r, snps = AssociationCalculation.association(gene, context, return_snps=True) results.append(r) snps_found.update(snps) reporter.update( len(snps_found), "%d %% of model's snps found so far in the gwas study") if args.additional_output: stats_ = AssociationCalculation.additional_stats(gene, context) additional.append(stats_) reporter.update(len(snps_found), "%d %% of model's snps used", force=True) results = AssociationCalculation.dataframe_from_results(results) results = MetaxcanUtilities.format_output(results, context, args.remove_ens_version) if args.additional_output: additional = AssociationCalculation.dataframe_from_aditional_stats( additional) results = MetaxcanUtilities.merge_additional_output( results, additional, context, args.remove_ens_version) if args.output_file: Utilities.ensure_requisite_folders(args.output_file) results.to_csv(args.output_file, index=False) return results
def run(args): if os.path.exists(args.snp_covariance_output): logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output) return start = timer() logging.info("Loading models...") model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern) all_snps = model_manager.get_rsids() logging.info("processing genotype") for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps): logging.log(9, "Processing chromosome %s", str(chromosome)) covariance_results = pandas.DataFrame() context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager) genes = context.get_genes() reporter = Utilities.PercentReporter(9, len(genes)) reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome)) for i,gene in enumerate(genes): logging.log(6, "%d/%d:%s", i+1, len(genes), gene) cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene) cov_data = MatrixManager._flatten_matrix_data([cov_data]) cov_data = Utilities.to_dataframe(cov_data, GenotypeAnalysis.COVARIANCE_COLUMNS, to_numeric="ignore", fill_na="NA") covariance_results = pandas.concat([covariance_results, cov_data]) reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome)) reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome)) logging.log(9, "writing chromosome results") Utilities.save_dataframe(covariance_results, args.snp_covariance_output, mode="w" if chromosome ==1 else "a", header=chromosome==1) end = timer() logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
def run(args): if os.path.exists(args.snp_covariance_output): logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output) return start = timer() logging.info("Loading models...") model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern, name_filter=args.models_filter) all_snps = model_manager.get_rsids() Utilities.ensure_requisite_folders(args.snp_covariance_output) with gzip.open(args.snp_covariance_output, "w") as o: o.write("GENE\tRSID1\tRSID2\tVALUE\n") logging.info("processing genotype") for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps): logging.log(9, "Processing chromosome %s", str(chromosome)) context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager) genes = context.get_genes() reporter = Utilities.PercentReporter(9, len(genes)) reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome)) for i,gene in enumerate(genes): logging.log(6, "%d/%d:%s", i+1, len(genes), gene) cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene) cov_data = MatrixManager._flatten_matrix_data([cov_data]) for e in cov_data: l = "{}\t{}\t{}\t{}\n".format(e[0], e[1], e[2], e[3]) o.write(l) reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome)) reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome)) end = timer() logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
def resultsFromCovarianceFile(self, weight_db_logic): results = {} logging.info("Loading covariance file from %s", self.covariance) covariance_contents = MatrixUtilities.loadMatrixFromFile( self.covariance) beta_contents = Utilities.contentsWithPatternsFromFolder( self.folder_beta, []) zscore_calculation, normalization = self.selectMethod( self.folder_beta, beta_contents, covariance_contents, weight_db_logic) total_entries = len(covariance_contents) reporter = Utilities.PercentReporter(logging.INFO, total_entries) i = 0 for beta_name in beta_contents: logging.info("Processing %s", beta_name) beta_path = os.path.join(self.folder_beta, beta_name) beta_sets = KeyedDataSet.KeyedDataSetFileUtilities.loadDataSetsFromCompressedFile( beta_path, header="") beta_sets = {set.name: set for set in beta_sets} key, check = beta_sets.iteritems().next() normalization.update(beta_sets) for gene, entry in covariance_contents.iteritems(): #So, new covariance files might actually have more genes than those in the database if not gene in weight_db_logic.weights_by_gene: logging.log(8, "Gene %s not in weights", gene) continue weights = weight_db_logic.weights_by_gene[gene] process = False for rsid, weight in weights.iteritems(): if rsid in check.values_by_key: process = True break if not process: logging.log(5, "No rsid in beta file for %s", gene) continue if gene in results: logging.info("Gene %s already processed", gene) continue covariance_matrix = entry[0] valid_rsids = entry[1] logging.log(7, "Calculating z score for %s", gene) pre_zscore, n, VAR_g, effect_size = zscore_calculation( gene, weights, beta_sets, covariance_matrix, valid_rsids) results[gene] = self.buildEntry(gene, weight_db_logic, weights, pre_zscore, n, VAR_g, effect_size) i += 1 reporter.update( i, "%d %% of model's snp information found so far in the gwas study" ) # proxied by percenteage of genes #second pass, for genes not in any beta file self.fillBlanks(results, covariance_contents, weight_db_logic, zscore_calculation) normalization_constant = normalization.calculateNormalization() return results, normalization_constant
def run(args): start = timer() if args.prediction_output: if os.path.exists(args.prediction_output[0]): logging.info( "Prediction output exists. Move or remove if you want this ran again." ) return Utilities.ensure_requisite_folders(args.prediction_output[0]) if args.prediction_summary_output: if os.path.exists(args.prediction_summary_output): logging.info( "Summary output exists. Move or remove if you want this ran again." ) return Utilities.ensure_requisite_folders(args.prediction_output[0]) logging.info("Loading samples") samples = load_samples(args) logging.info("Loading model") model, weights, extra = model_structure(args) variant_mapping = get_variant_mapping(args, weights) logging.info("Preparing genotype dosages") dosage_source = dosage_generator(args, variant_mapping, weights) logging.info("Processing genotypes") dcapture = [] reporter = Utilities.PercentReporter(logging.INFO, len(set(weights.rsid.values))) snps_found = set() with prepare_prediction(args, extra, samples) as results: for i, e in enumerate(dosage_source): if args.stop_at_variant and i > args.stop_at_variant: break var_id = e[GF.RSID] logging.log(8, "variant %i:%s", i, var_id) if var_id in model: s = model[var_id] ref_allele, alt_allele = e[GF.REF_ALLELE], e[GF.ALT_ALLELE] allele_align, strand_align = GWASAndModels.match_alleles( ref_allele, alt_allele, s[0], s[1]) if not allele_align or not strand_align: continue dosage = e[GF.FIRST_DOSAGE:] if allele_align == -1: dosage = tuple(map(lambda x: 2 - x, dosage)) dosage = numpy.array(dosage, dtype=numpy.float) snps_found.add(var_id) for gene, weight in s[2].items(): results.update(gene, dosage, weight) if args.capture: dcapture.append((gene, weight, var_id, s[0], s[1], ref_allele, alt_allele, strand_align, allele_align) + e[GF.FIRST_DOSAGE:]) reporter.update(len(snps_found), "%d %% of models' snps used") reporter.update(len(snps_found), "%d %% of models' snps used", force=True) if args.capture: logging.info("Saving data capture") Utilities.ensure_requisite_folders(args.capture) with gzip.open(args.capture, "w") as f: header = "gene\tweight\tvariant_id\tref_allele\teff_allele\ta0\ta1\tstrand_align\tallele_align\t" + "\t".join( samples.IID.values) + "\n" f.write(header.encode()) for c in dcapture: l = "\t".join(map(str, c)) + "\n" f.write(l.encode()) if args.prediction_output and len(args.prediction_output) < 2: logging.info("Storing prediction") results.store_prediction() if args.prediction_summary_output: logging.info("Saving summary") summary = results.summary() Utilities.save_dataframe(summary, args.prediction_summary_output) end = timer() logging.info("Successfully predicted expression in %s seconds" % (str(end - start))) return results