def run(args):
start = timer()
if os.path.exists(args.output):
logging.info(
"%s already exists, you have to move it or delete it if you want it done again",
args.output)
return
logging.info("Creating context")
context = CrossModelUtilities.context_from_args(args)
results = []
n_genes = context.get_n_genes()
reporter = Utilities.PercentReporter(logging.INFO, n_genes)
logging.info("Processing")
reporter.update(0, "%d %% of model's genes processed so far")
for i, gene in enumerate(context.get_genes()):
logging.log(7, "Gene %d/%d: %s", i + 1, n_genes, gene)
result = JointAnalysis.joint_analysis(context, gene)
results.append(result)
reporter.update(i, "%d %% of model's genes processed so far")
results = JointAnalysis.format_results(results)
Utilities.ensure_requisite_folders(args.output)
results.to_csv(args.output, index=False, sep="\t")
end = timer()
logging.info("Ran multi tissue in %s seconds" % (str(end - start)))
def run(args):
start = timer()
if os.path.exists(args.output):
logging.info("%s already exists, you have to move it or delete it if you want it done again", args.output)
return
if (args.hdf5_expression_file and args.expression_file) or \
(not args.hdf5_expression_file and not args.expression_file):
logging.info("Provide either hdf5 expression file or plain text expression file")
return
with PrediXcanUtilities.p_context_from_args(args) as context:
genes = context.get_genes()
n_genes = len(genes)
reporter = Utilities.PercentReporter(logging.INFO, n_genes)
reporter.update(0, "%d %% of model's genes processed so far", force=True)
results = []
for i,gene in enumerate(genes):
logging.log(7, "Processing gene %s", gene)
r = PrediXcanAssociation.predixcan_association(gene, context)
results.append(r)
reporter.update(i, "%d %% of model's genes processed so far")
reporter.update(i, "%d %% of model's genes processed so far")
results = PrediXcanAssociation.dataframe_from_results(results)
results = results.fillna("NA")
results = results.sort_values(by="pvalue")
Utilities.save_dataframe(results, args.output)
end = timer()
logging.info("Ran multi tissue predixcan in %s seconds" % (str(end - start)))
def run(args, _gwas=None):
start = timer()
if not args.overwrite and os.path.exists(args.output_file):
logging.info("%s already exists, move it or delete it if you want it done again", args.output_file)
return
logging.info("Started metaxcan association")
context = MetaxcanUtilities.build_context(args, _gwas)
model_snps = context.get_model_snps()
total_snps = len(model_snps)
snps_found=set()
reporter = Utilities.PercentReporter(logging.INFO, total_snps)
i_genes, i_snps = context.get_data_intersection()
results = []
for gene in i_genes:
r, snps = AssociationCalculation.association(gene, context, return_snps=True)
results.append(r)
snps_found.update(snps)
reporter.update(len(snps_found), "%d %% of model's snps found so far in the gwas study")
Utilities.ensure_requisite_folders(args.output_file)
reporter.update(len(snps_found), "%d %% of model's snps used", force=True)
results = AssociationCalculation.dataframe_from_results(zip(*results))
results = MetaxcanUtilities.format_output(results, context, args.keep_ens_version)
results.to_csv(args.output_file, index=False)
end = timer()
logging.info("Sucessfully processed metaxcan association in %s seconds"%(str(end - start)))
def run_metaxcan(args, context):
logging.info("Started metaxcan association")
model_snps = context.get_model_snps()
total_snps = len(model_snps)
snps_found=set()
reporter = Utilities.PercentReporter(logging.INFO, total_snps)
i_genes, i_snps = context.get_data_intersection()
results = []
for gene in i_genes:
logging.log(7, "Processing gene %s", gene)
r, snps = AssociationCalculation.association(gene, context, return_snps=True)
results.append(r)
snps_found.update(snps)
reporter.update(len(snps_found), "%d %% of model's snps found so far in the gwas study")
reporter.update(len(snps_found), "%d %% of model's snps used", force=True)
results = AssociationCalculation.dataframe_from_results(results)
results = MetaxcanUtilities.format_output(results, context, args.remove_ens_version)
if args.output_file:
Utilities.ensure_requisite_folders(args.output_file)
results.to_csv(args.output_file, index=False)
return results
def run(args):
start = timer()
if os.path.exists(args.output):
logging.info(
"%s already exists, you have to move it or delete it if you want it done again",
args.output)
return
if (args.hdf5_expression_folder and args.expression_folder) or \
(not args.hdf5_expression_folder and not args.expression_folder):
logging.info(
"Provide either hdf5 expression folder or plain text expression folder"
)
return
with MultiPrediXcanUtilities.mp_context_from_args(args) as context:
genes = context.get_genes()
n_genes = len(genes)
reporter = Utilities.PercentReporter(logging.INFO, n_genes)
reporter.update(0,
"%d %% of model's genes processed so far",
force=True)
results = []
callbacks = {}
if args.coefficient_output:
callbacks["coefficient"] = MultiPrediXcanAssociation.SaveCoefs()
if args.loadings_output:
callbacks["loadings"] = MultiPrediXcanAssociation.SaveLoadings()
for i, gene in enumerate(genes):
logging.log(7, "Processing gene %i/%i: %s", i + 1, n_genes, gene)
r = MultiPrediXcanAssociation.multi_predixcan_association(
gene, context, callbacks.values())
results.append(r)
reporter.update(i, "%d %% of model's genes processed so far")
reporter.update(i, "%d %% of model's genes processed so far")
results = MultiPrediXcanAssociation.dataframe_from_results(
results, context)
results = results.fillna("NA")
results = results.sort_values(by="pvalue")
Utilities.save_dataframe(results, args.output)
if args.coefficient_output:
Utilities.save_dataframe(callbacks["coefficient"].get(),
args.coefficient_output)
if args.loadings_output:
Utilities.save_dataframe(callbacks["loadings"].get(),
args.loadings_output)
end = timer()
logging.info("Ran multi tissue predixcan in %s seconds" %
(str(end - start)))
def run_metaxcan(args, context):
logging.info("Started metaxcan association")
model_snps = context.get_model_snps()
total_snps = len(model_snps)
snps_found = set()
reporter = Utilities.PercentReporter(logging.INFO, total_snps)
i_genes, i_snps = context.get_data_intersection()
results = []
additional = []
for i, gene in enumerate(i_genes):
if args.MAX_R and i + 1 > args.MAX_R:
logging.log("Early exit condition met")
break
logging.log(9, "Processing gene %i:%s", i, gene)
r, snps = AssociationCalculation.association(gene,
context,
return_snps=True)
results.append(r)
snps_found.update(snps)
reporter.update(
len(snps_found),
"%d %% of model's snps found so far in the gwas study")
if args.additional_output:
stats_ = AssociationCalculation.additional_stats(gene, context)
additional.append(stats_)
reporter.update(len(snps_found), "%d %% of model's snps used", force=True)
results = AssociationCalculation.dataframe_from_results(results)
results = MetaxcanUtilities.format_output(results, context,
args.remove_ens_version)
if args.additional_output:
additional = AssociationCalculation.dataframe_from_aditional_stats(
additional)
results = MetaxcanUtilities.merge_additional_output(
results, additional, context, args.remove_ens_version)
if args.output_file:
Utilities.ensure_requisite_folders(args.output_file)
results.to_csv(args.output_file, index=False)
return results
def run(args):
if os.path.exists(args.snp_covariance_output):
logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output)
return
start = timer()
logging.info("Loading models...")
model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern)
all_snps = model_manager.get_rsids()
logging.info("processing genotype")
for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps):
logging.log(9, "Processing chromosome %s", str(chromosome))
covariance_results = pandas.DataFrame()
context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager)
genes = context.get_genes()
reporter = Utilities.PercentReporter(9, len(genes))
reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome))
for i,gene in enumerate(genes):
logging.log(6, "%d/%d:%s", i+1, len(genes), gene)
cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene)
cov_data = MatrixManager._flatten_matrix_data([cov_data])
cov_data = Utilities.to_dataframe(cov_data, GenotypeAnalysis.COVARIANCE_COLUMNS, to_numeric="ignore", fill_na="NA")
covariance_results = pandas.concat([covariance_results, cov_data])
reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome))
reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome))
logging.log(9, "writing chromosome results")
Utilities.save_dataframe(covariance_results, args.snp_covariance_output,
mode="w" if chromosome ==1 else "a",
header=chromosome==1)
end = timer()
logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
def run(args):
if os.path.exists(args.snp_covariance_output):
logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output)
return
start = timer()
logging.info("Loading models...")
model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern, name_filter=args.models_filter)
all_snps = model_manager.get_rsids()
Utilities.ensure_requisite_folders(args.snp_covariance_output)
with gzip.open(args.snp_covariance_output, "w") as o:
o.write("GENE\tRSID1\tRSID2\tVALUE\n")
logging.info("processing genotype")
for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps):
logging.log(9, "Processing chromosome %s", str(chromosome))
context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager)
genes = context.get_genes()
reporter = Utilities.PercentReporter(9, len(genes))
reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome))
for i,gene in enumerate(genes):
logging.log(6, "%d/%d:%s", i+1, len(genes), gene)
cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene)
cov_data = MatrixManager._flatten_matrix_data([cov_data])
for e in cov_data:
l = "{}\t{}\t{}\t{}\n".format(e[0], e[1], e[2], e[3])
o.write(l)
reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome))
reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome))
end = timer()
logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
def resultsFromCovarianceFile(self, weight_db_logic):
results = {}
logging.info("Loading covariance file from %s", self.covariance)
covariance_contents = MatrixUtilities.loadMatrixFromFile(
self.covariance)
beta_contents = Utilities.contentsWithPatternsFromFolder(
self.folder_beta, [])
zscore_calculation, normalization = self.selectMethod(
self.folder_beta, beta_contents, covariance_contents,
weight_db_logic)
total_entries = len(covariance_contents)
reporter = Utilities.PercentReporter(logging.INFO, total_entries)
i = 0
for beta_name in beta_contents:
logging.info("Processing %s", beta_name)
beta_path = os.path.join(self.folder_beta, beta_name)
beta_sets = KeyedDataSet.KeyedDataSetFileUtilities.loadDataSetsFromCompressedFile(
beta_path, header="")
beta_sets = {set.name: set for set in beta_sets}
key, check = beta_sets.iteritems().next()
normalization.update(beta_sets)
for gene, entry in covariance_contents.iteritems():
#So, new covariance files might actually have more genes than those in the database
if not gene in weight_db_logic.weights_by_gene:
logging.log(8, "Gene %s not in weights", gene)
continue
weights = weight_db_logic.weights_by_gene[gene]
process = False
for rsid, weight in weights.iteritems():
if rsid in check.values_by_key:
process = True
break
if not process:
logging.log(5, "No rsid in beta file for %s", gene)
continue
if gene in results:
logging.info("Gene %s already processed", gene)
continue
covariance_matrix = entry[0]
valid_rsids = entry[1]
logging.log(7, "Calculating z score for %s", gene)
pre_zscore, n, VAR_g, effect_size = zscore_calculation(
gene, weights, beta_sets, covariance_matrix, valid_rsids)
results[gene] = self.buildEntry(gene, weight_db_logic, weights,
pre_zscore, n, VAR_g,
effect_size)
i += 1
reporter.update(
i,
"%d %% of model's snp information found so far in the gwas study"
) # proxied by percenteage of genes
#second pass, for genes not in any beta file
self.fillBlanks(results, covariance_contents, weight_db_logic,
zscore_calculation)
normalization_constant = normalization.calculateNormalization()
return results, normalization_constant
def run(args):
start = timer()
if args.prediction_output:
if os.path.exists(args.prediction_output[0]):
logging.info(
"Prediction output exists. Move or remove if you want this ran again."
)
return
Utilities.ensure_requisite_folders(args.prediction_output[0])
if args.prediction_summary_output:
if os.path.exists(args.prediction_summary_output):
logging.info(
"Summary output exists. Move or remove if you want this ran again."
)
return
Utilities.ensure_requisite_folders(args.prediction_output[0])
logging.info("Loading samples")
samples = load_samples(args)
logging.info("Loading model")
model, weights, extra = model_structure(args)
variant_mapping = get_variant_mapping(args, weights)
logging.info("Preparing genotype dosages")
dosage_source = dosage_generator(args, variant_mapping, weights)
logging.info("Processing genotypes")
dcapture = []
reporter = Utilities.PercentReporter(logging.INFO,
len(set(weights.rsid.values)))
snps_found = set()
with prepare_prediction(args, extra, samples) as results:
for i, e in enumerate(dosage_source):
if args.stop_at_variant and i > args.stop_at_variant:
break
var_id = e[GF.RSID]
logging.log(8, "variant %i:%s", i, var_id)
if var_id in model:
s = model[var_id]
ref_allele, alt_allele = e[GF.REF_ALLELE], e[GF.ALT_ALLELE]
allele_align, strand_align = GWASAndModels.match_alleles(
ref_allele, alt_allele, s[0], s[1])
if not allele_align or not strand_align:
continue
dosage = e[GF.FIRST_DOSAGE:]
if allele_align == -1:
dosage = tuple(map(lambda x: 2 - x, dosage))
dosage = numpy.array(dosage, dtype=numpy.float)
snps_found.add(var_id)
for gene, weight in s[2].items():
results.update(gene, dosage, weight)
if args.capture:
dcapture.append((gene, weight, var_id, s[0], s[1],
ref_allele, alt_allele, strand_align,
allele_align) + e[GF.FIRST_DOSAGE:])
reporter.update(len(snps_found), "%d %% of models' snps used")
reporter.update(len(snps_found), "%d %% of models' snps used", force=True)
if args.capture:
logging.info("Saving data capture")
Utilities.ensure_requisite_folders(args.capture)
with gzip.open(args.capture, "w") as f:
header = "gene\tweight\tvariant_id\tref_allele\teff_allele\ta0\ta1\tstrand_align\tallele_align\t" + "\t".join(
samples.IID.values) + "\n"
f.write(header.encode())
for c in dcapture:
l = "\t".join(map(str, c)) + "\n"
f.write(l.encode())
if args.prediction_output and len(args.prediction_output) < 2:
logging.info("Storing prediction")
results.store_prediction()
if args.prediction_summary_output:
logging.info("Saving summary")
summary = results.summary()
Utilities.save_dataframe(summary, args.prediction_summary_output)
end = timer()
logging.info("Successfully predicted expression in %s seconds" %
(str(end - start)))
return results
