def run(args):
start = timer()
if os.path.exists(args.output):
logging.info("%s already exists, you have to move it or delete it if you want it done again", args.output)
return
if (args.hdf5_expression_file and args.expression_file) or \
(not args.hdf5_expression_file and not args.expression_file):
logging.info("Provide either hdf5 expression file or plain text expression file")
return
with PrediXcanUtilities.p_context_from_args(args) as context:
genes = context.get_genes()
n_genes = len(genes)
reporter = Utilities.PercentReporter(logging.INFO, n_genes)
reporter.update(0, "%d %% of model's genes processed so far", force=True)
results = []
for i,gene in enumerate(genes):
logging.log(7, "Processing gene %s", gene)
r = PrediXcanAssociation.predixcan_association(gene, context)
results.append(r)
reporter.update(i, "%d %% of model's genes processed so far")
reporter.update(i, "%d %% of model's genes processed so far")
results = PrediXcanAssociation.dataframe_from_results(results)
results = results.fillna("NA")
results = results.sort_values(by="pvalue")
Utilities.save_dataframe(results, args.output)
end = timer()
logging.info("Ran multi tissue predixcan in %s seconds" % (str(end - start)))
def run(args):
if os.path.exists(args.snp_covariance_output):
logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output)
return
start = timer()
logging.info("Loading models...")
model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern)
all_snps = model_manager.get_rsids()
logging.info("processing genotype")
for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps):
logging.log(9, "Processing chromosome %s", str(chromosome))
covariance_results = pandas.DataFrame()
context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager)
genes = context.get_genes()
reporter = Utilities.PercentReporter(9, len(genes))
reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome))
for i,gene in enumerate(genes):
logging.log(6, "%d/%d:%s", i+1, len(genes), gene)
cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene)
cov_data = MatrixManager._flatten_matrix_data([cov_data])
cov_data = Utilities.to_dataframe(cov_data, GenotypeAnalysis.COVARIANCE_COLUMNS, to_numeric="ignore", fill_na="NA")
covariance_results = pandas.concat([covariance_results, cov_data])
reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome))
reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome))
logging.log(9, "writing chromosome results")
Utilities.save_dataframe(covariance_results, args.snp_covariance_output,
mode="w" if chromosome ==1 else "a",
header=chromosome==1)
end = timer()
logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
def run(args):
start = timer()
if os.path.exists(args.output):
logging.info(
"%s already exists, you have to move it or delete it if you want it done again",
args.output)
return
if (args.hdf5_expression_folder and args.expression_folder) or \
(not args.hdf5_expression_folder and not args.expression_folder):
logging.info(
"Provide either hdf5 expression folder or plain text expression folder"
)
return
with MultiPrediXcanUtilities.mp_context_from_args(args) as context:
genes = context.get_genes()
n_genes = len(genes)
reporter = Utilities.PercentReporter(logging.INFO, n_genes)
reporter.update(0,
"%d %% of model's genes processed so far",
force=True)
results = []
callbacks = {}
if args.coefficient_output:
callbacks["coefficient"] = MultiPrediXcanAssociation.SaveCoefs()
if args.loadings_output:
callbacks["loadings"] = MultiPrediXcanAssociation.SaveLoadings()
for i, gene in enumerate(genes):
logging.log(7, "Processing gene %i/%i: %s", i + 1, n_genes, gene)
r = MultiPrediXcanAssociation.multi_predixcan_association(
gene, context, callbacks.values())
results.append(r)
reporter.update(i, "%d %% of model's genes processed so far")
reporter.update(i, "%d %% of model's genes processed so far")
results = MultiPrediXcanAssociation.dataframe_from_results(
results, context)
results = results.fillna("NA")
results = results.sort_values(by="pvalue")
Utilities.save_dataframe(results, args.output)
if args.coefficient_output:
Utilities.save_dataframe(callbacks["coefficient"].get(),
args.coefficient_output)
if args.loadings_output:
Utilities.save_dataframe(callbacks["loadings"].get(),
args.loadings_output)
end = timer()
logging.info("Ran multi tissue predixcan in %s seconds" %
(str(end - start)))
def run(args):
start = timer()
folder, prefix = os.path.split(args.output_prefix)
results_name = args.output_prefix + "__mt_results.txt"
predixcan_results_name = args.output_prefix + "__p_results.txt"
additional_name = args.output_prefix + "__additional.txt"
if os.path.exists(results_name):
logging.info(
"%s already exists, you have to move it or delete it if you want it done again",
results_name)
return
#for reproducibility
numpy.random.seed(100)
results = []
additional = []
predixcan_results = []
n_max = args.max_n_results
logging.info("Acquiring context")
with MultiPredixcanSimulations.context_from_args(args) as context:
logging.info("processing")
_c, _cp, _e = context.get_mp_simulation(None)
for i, gene in enumerate(context.get_genes()):
if n_max and i + 1 > n_max:
logging.info("Max runs met")
break
logging.log(9, "%d Gene %s", i, gene)
r, add, p = MultiPredixcanSimulations.simulate(gene, context)
if r is None:
logging.log(9, "%s could not be simulated", gene)
continue
results.append(r)
additional.append(add)
if p is not None:
predixcan_results.append(p)
results = MultiPrediXcanAssociation.dataframe_from_results(
results, _c).sort_values(by="pvalue")
additional = pandas.concat(additional)
Utilities.ensure_requisite_folders(results_name)
Utilities.save_dataframe(results, results_name)
Utilities.save_dataframe(additional, additional_name)
if len(predixcan_results):
predixcan_results = pandas.concat(predixcan_results)
Utilities.save_dataframe(predixcan_results, predixcan_results_name)
logging.info("Finished")
def run(args):
start = timer()
folder, prefix = os.path.split(args.output_prefix)
results_name = args.output_prefix + "__mt_results.txt"
predixcan_results_name = args.output_prefix + "__p_results.txt"
additional_name = args.output_prefix + "__additional.txt"
if os.path.exists(results_name):
logging.info("%s already exists, you have to move it or delete it if you want it done again", results_name)
return
#for reproducibility
numpy.random.seed(100)
results = []
additional = []
predixcan_results = []
n_max = args.max_n_results
logging.info("Acquiring context")
with MultiPredixcanSimulations.context_from_args(args) as context:
logging.info("processing")
_c, _cp, _e = context.get_mp_simulation(None)
for i, gene in enumerate(context.get_genes()):
if n_max and i+1>n_max:
logging.info("Max runs met")
break
logging.log(9, "%d Gene %s", i, gene)
r, add, p = MultiPredixcanSimulations.simulate(gene, context)
if r is None:
logging.log(9, "%s could not be simulated", gene)
continue
results.append(r)
additional.append(add)
if p is not None:
predixcan_results.append(p)
results = MultiPrediXcanAssociation.dataframe_from_results(results, _c).sort_values(by="pvalue")
additional = pandas.concat(additional)
Utilities.ensure_requisite_folders(results_name)
Utilities.save_dataframe(results, results_name)
Utilities.save_dataframe(additional, additional_name)
if len(predixcan_results):
predixcan_results = pandas.concat(predixcan_results)
Utilities.save_dataframe(predixcan_results, predixcan_results_name)
logging.info("Finished")
def run(args):
start = timer()
if args.prediction_output:
if os.path.exists(args.prediction_output[0]):
logging.info(
"Prediction output exists. Move or remove if you want this ran again."
)
return
Utilities.ensure_requisite_folders(args.prediction_output[0])
if args.prediction_summary_output:
if os.path.exists(args.prediction_summary_output):
logging.info(
"Summary output exists. Move or remove if you want this ran again."
)
return
Utilities.ensure_requisite_folders(args.prediction_output[0])
logging.info("Loading samples")
samples = load_samples(args)
logging.info("Loading model")
model, weights, extra = model_structure(args)
variant_mapping = get_variant_mapping(args, weights)
logging.info("Preparing genotype dosages")
dosage_source = dosage_generator(args, variant_mapping, weights)
logging.info("Processing genotypes")
dcapture = []
reporter = Utilities.PercentReporter(logging.INFO,
len(set(weights.rsid.values)))
snps_found = set()
with prepare_prediction(args, extra, samples) as results:
for i, e in enumerate(dosage_source):
if args.stop_at_variant and i > args.stop_at_variant:
break
var_id = e[GF.RSID]
logging.log(8, "variant %i:%s", i, var_id)
if var_id in model:
s = model[var_id]
ref_allele, alt_allele = e[GF.REF_ALLELE], e[GF.ALT_ALLELE]
allele_align, strand_align = GWASAndModels.match_alleles(
ref_allele, alt_allele, s[0], s[1])
if not allele_align or not strand_align:
continue
dosage = e[GF.FIRST_DOSAGE:]
if allele_align == -1:
dosage = tuple(map(lambda x: 2 - x, dosage))
dosage = numpy.array(dosage, dtype=numpy.float)
snps_found.add(var_id)
for gene, weight in s[2].items():
results.update(gene, dosage, weight)
if args.capture:
dcapture.append((gene, weight, var_id, s[0], s[1],
ref_allele, alt_allele, strand_align,
allele_align) + e[GF.FIRST_DOSAGE:])
reporter.update(len(snps_found), "%d %% of models' snps used")
reporter.update(len(snps_found), "%d %% of models' snps used", force=True)
if args.capture:
logging.info("Saving data capture")
Utilities.ensure_requisite_folders(args.capture)
with gzip.open(args.capture, "w") as f:
header = "gene\tweight\tvariant_id\tref_allele\teff_allele\ta0\ta1\tstrand_align\tallele_align\t" + "\t".join(
samples.IID.values) + "\n"
f.write(header.encode())
for c in dcapture:
l = "\t".join(map(str, c)) + "\n"
f.write(l.encode())
if args.prediction_output and len(args.prediction_output) < 2:
logging.info("Storing prediction")
results.store_prediction()
if args.prediction_summary_output:
logging.info("Saving summary")
summary = results.summary()
Utilities.save_dataframe(summary, args.prediction_summary_output)
end = timer()
logging.info("Successfully predicted expression in %s seconds" %
(str(end - start)))
return results
def store_prediction(self):
logging.info("Saving prediction as a text file")
d = pandas.DataFrame(self.genes)
result = pandas.concat([self.samples, d], axis=1, sort=False)
Utilities.save_dataframe(result, self.output_path)
