def test_load(self):
        t = PredictionModel.ModelDB("tests/_td/dbs/test_1.db")
        extra = t.load_extra()
        self.assertEqual(len(extra), 6)

        e_e = zip(*(SampleData.sample_extra_2()))
        self.assertEqual(extra[PredictionModel.WDBEQF.GENE],
                         e_e[PredictionModel.WDBEQF.GENE])
        self.assertEqual(extra[PredictionModel.WDBEQF.GENE_NAME],
                         e_e[PredictionModel.WDBEQF.GENE_NAME])
        self.assertEqual(extra[PredictionModel.WDBEQF.N_SNP_IN_MODEL],
                         e_e[PredictionModel.WDBEQF.N_SNP_IN_MODEL])
        self.assertEqual(extra[PredictionModel.WDBEQF.PRED_PERF_R2],
                         e_e[PredictionModel.WDBEQF.PRED_PERF_R2])
        self.assertEqual(extra[PredictionModel.WDBEQF.PRED_PERF_PVAL],
                         e_e[PredictionModel.WDBEQF.PRED_PERF_PVAL])
        self.assertEqual(extra[PredictionModel.WDBEQF.PRED_PERF_QVAL],
                         e_e[PredictionModel.WDBEQF.PRED_PERF_QVAL])

        weights = t.load_weights()
        self.assertEqual(len(weights), 5)

        e_w = zip(*(SampleData.sample_weights_2()))
        self.assertEqual(weights[PredictionModel.WDBQF.RSID],
                         e_w[PredictionModel.WDBQF.RSID])
        self.assertEqual(weights[PredictionModel.WDBQF.GENE],
                         e_w[PredictionModel.WDBQF.GENE])
        self.assertEqual(weights[PredictionModel.WDBQF.WEIGHT],
                         e_w[PredictionModel.WDBQF.WEIGHT])
        self.assertEqual(weights[PredictionModel.WDBQF.REF_ALLELE],
                         e_w[PredictionModel.WDBQF.REF_ALLELE])
        self.assertEqual(weights[PredictionModel.WDBQF.EFF_ALLELE],
                         e_w[PredictionModel.WDBQF.EFF_ALLELE])
    def test_model_manager(self):
        model_manager = PredictionModel.load_model_manager("tests/_td/dbs_2")
        weights = get_weights_in_models("tests/_td/dbs_2")
        models_ = weights[["model", "gene"]].drop_duplicates()
        for t in models_.itertuples():
            compare_model_manager_models_to_weights(self, model_manager,
                                                    weights, t.model, t.gene)
        #
        self.assertEqual(
            model_manager.get_genes(),
            {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K'})
        self.assertEqual(model_manager.get_rsids(), set(weights.rsid))
        self.assertEqual(model_manager.get_model_labels(),
                         {"model_sim_1", "model_sim_2"})
        #
        for rsid in set(weights.rsid):
            self.assertEqual(model_manager.snp_keys[rsid],
                             set(weights[weights.rsid == rsid].gene))
        #
        for gene in set(weights.gene):
            w = weights[weights.gene == gene].set_index(
                ["model", "rsid"])[["weight", "eff_allele", "ref_allele"]]
            m = model_manager.get_models(gene).rename(
                columns={
                    "effect_allele": "eff_allele",
                    "non_effect_allele": "ref_allele"
                })
            for m_ in set(m.index.get_level_values(0)):
                _compare(self, m.loc[m_], w.loc[m_])

        #
        for gene in set(weights.gene):
            self.assertEqual(model_manager.get_model_labels(gene),
                             set(weights[weights.gene == gene].model))
示例#3
0
def run(args):
    start = timer()
    validate(args)

    if args.gwas_folder:
        regexp = re.compile(
            args.gwas_file_pattern) if args.gwas_file_pattern else None
        names = Utilities.contentsWithRegexpFromFolder(args.gwas_folder,
                                                       regexp)
        names.sort(
        )  #cosmetic, because different filesystems/OS yield folders in different order

        if len(names) == 0:
            msg = "No GWAS files found on %s with pattern %s" % (
                args.gwas_folder,
                args.gwas_file_pattern,
            )
            raise Exceptions.ReportableException(msg)
    else:
        names = [args.gwas_file]

    gwas_format = GWASUtilities.gwas_format_from_args(args)
    GWAS.validate_format_basic(gwas_format)
    GWAS.validate_format_for_strict(gwas_format)
    model = PredictionModel.load_model(
        args.model_db_path,
        args.model_db_snp_key) if args.model_db_path else None

    if args.output_folder:
        if not os.path.exists(args.output_folder):
            os.makedirs(args.output_folder)

        for name in names:
            output_path = os.path.join(args.output_folder, name)
            if not ".gz" in output_path:
                output_path += ".gz"
            if os.path.exists(output_path):
                logging.info(
                    "%s already exists, delete it if you want it to be done again",
                    output_path)
                continue

            b = build_betas(args, model, gwas_format, name)
            c = "gzip" if ".gz" in name else None
            b.to_csv(output_path, sep="\t", index=False, compression=c)
        end = timer()
        logging.info("Successfully ran GWAS input processing in %s seconds" %
                     (str(end - start)))
    else:
        r = pandas.DataFrame()
        for name in names:
            b = build_betas(args, model, gwas_format, name)
            r = pandas.concat([r, b])
        end = timer()
        logging.info("Successfully parsed input gwas in %s seconds" %
                     (str(end - start)))

        return r
    def test_optimized_model_manager_gtex(self):
        model_manager = PredictionModel.load_model_manager(
            "tests/_td/dbs_3", Klass=PredictionModel._ModelManager)
        weights = get_weights_in_models("tests/_td/dbs_3")
        weights.model = weights.model.str.extract("TW_(.*)_0.5", expand=False)

        #
        _assert_optimized_manager(
            self, model_manager, weights,
            {"ENSG00000107937.14", "ENSG00000107959.11", "ENSG00000234745.5"})
 def test_snps_in_db(self):
     expected = {
         "rs245915", "rs245913", "rs245909", "rs245906", "rs10486599",
         "rs144012121", "rs117887801", "rs542000", "rs544632", "rs498475",
         "rs849327", "rs849336", "rs849335", "rs1513272", "rs849135",
         "rs849134", "rs860262", "rs849133", "rs1635852", "rs864745",
         "rs112751321", "rs144273091", "rs117462481", "rs149305679",
         "rs643036", "rs1937888", "rs17155745", "rs62626328"
     }
     actual = PredictionModel.snps_in_db("tests/_td/dbs/test_2.db")
     self.assertEqual(actual, expected)
    def test_optimized_model_manager_gtex_trimmed(self):
        model_manager = PredictionModel.load_model_manager(
            "tests/_td/dbs_3",
            trim_ensemble_version=True,
            Klass=PredictionModel._ModelManager)
        weights = get_weights_in_models("tests/_td/dbs_3")
        weights.model = weights.model.str.extract("TW_(.*)_0.5", expand=False)
        weights.gene = weights.gene.str.split(".").str.get(0)

        #
        _assert_optimized_manager(
            self, model_manager, weights,
            {"ENSG00000107937", "ENSG00000107959", "ENSG00000234745"})
    def test_load_model(self):
        snp_model = PredictionModel.load_model("tests/_td/dbs/test_1.db")

        e_e = SampleData.dataframe_from_extra(SampleData.sample_extra_2())
        numpy.testing.assert_array_equal(snp_model.extra[PredictionModel.WDBEQF.K_GENE], e_e[PredictionModel.WDBEQF.K_GENE])
        numpy.testing.assert_array_equal(snp_model.extra[PredictionModel.WDBEQF.K_GENE_NAME], e_e[PredictionModel.WDBEQF.K_GENE_NAME])
        numpy.testing.assert_array_equal(snp_model.extra[PredictionModel.WDBEQF.K_N_SNP_IN_MODEL], e_e[PredictionModel.WDBEQF.K_N_SNP_IN_MODEL])
        numpy.testing.assert_array_equal(snp_model.extra[PredictionModel.WDBEQF.K_PRED_PERF_R2], e_e[PredictionModel.WDBEQF.K_PRED_PERF_R2])
        numpy.testing.assert_array_equal(snp_model.extra[PredictionModel.WDBEQF.K_PRED_PERF_PVAL], e_e[PredictionModel.WDBEQF.K_PRED_PERF_PVAL])
        numpy.testing.assert_array_equal(snp_model.extra[PredictionModel.WDBEQF.K_PRED_PERF_QVAL], e_e[PredictionModel.WDBEQF.K_PRED_PERF_QVAL])

        e_w = SampleData.dataframe_from_weights(SampleData.sample_weights_2())
        numpy.testing.assert_array_equal(snp_model.weights[PredictionModel.WDBQF.K_RSID], e_w[PredictionModel.WDBQF.K_RSID])
        numpy.testing.assert_array_equal(snp_model.weights[PredictionModel.WDBQF.K_GENE], e_w[PredictionModel.WDBQF.K_GENE])
        numpy.testing.assert_array_equal(snp_model.weights[PredictionModel.WDBQF.K_WEIGHT], e_w[PredictionModel.WDBQF.K_WEIGHT])
        numpy.testing.assert_array_equal(snp_model.weights[PredictionModel.WDBQF.K_NON_EFFECT_ALLELE], e_w[PredictionModel.WDBQF.K_NON_EFFECT_ALLELE])
        numpy.testing.assert_array_equal(snp_model.weights[PredictionModel.WDBQF.K_EFFECT_ALLELE], e_w[PredictionModel.WDBQF.K_EFFECT_ALLELE])
示例#8
0
def run(args):
    start = timer()
    validate(args)

    if args.gwas_folder:
        regexp = re.compile(args.gwas_file_pattern) if args.gwas_file_pattern else  None
        names = Utilities.contentsWithRegexpFromFolder(args.gwas_folder, regexp)
        names.sort() #cosmetic, because different filesystems/OS yield folders in different order

        if len(names) == 0:
            msg = "No GWAS files found on %s with pattern %s" % (args.gwas_folder, args.gwas_file_pattern,)
            raise Exceptions.ReportableException(msg)
    else:
        names = [args.gwas_file]

    gwas_format = GWASUtilities.gwas_format_from_args(args)
    GWAS.validate_format_basic(gwas_format)
    GWAS.validate_format_for_strict(gwas_format)
    model = PredictionModel.load_model(args.model_db_path, args.model_db_snp_key) if args.model_db_path else None

    if args.output_folder:
        if not os.path.exists(args.output_folder):
            os.makedirs(args.output_folder)

        for name in names:
            output_path = os.path.join(args.output_folder, name)
            if not ".gz" in output_path:
                output_path += ".gz"
            if os.path.exists(output_path):
                logging.info("%s already exists, delete it if you want it to be done again", output_path)
                continue

            b = build_betas(args, model, gwas_format, name)
            c = "gzip" if ".gz" in name else None
            b.to_csv(output_path, sep="\t", index=False, compression=c)
        end = timer()
        logging.info("Successfully ran GWAS input processing in %s seconds" %(str(end - start)))
    else:
        r = pandas.DataFrame()
        for name in names:
            b = build_betas(args, model, gwas_format, name)
            r = pandas.concat([r,b])
        end = timer()
        logging.info("Successfully parsed input gwas in %s seconds"%(str(end-start)))

        return r
示例#9
0
def model_structure(args):
    model = PredictionModel.load_model(args.model_db_path,
                                       args.model_db_snp_key)
    m = {}
    weights, extra = model.weights, model.extra
    if args.sub_batches is not None and args.sub_batch is not None:
        logging.info("slicing models")
        extra = Utilities.sub_batch(extra, args.sub_batches, args.sub_batch)
        weights = weights[weights.gene.isin(extra.gene)].reset_index(drop=True)

    if args.only_entries:
        extra = extra[extra.gene.isin(set(args.only_entries))]
        weights = weights[weights.gene.isin(set(args.only_entries))]

    for i in weights.itertuples():
        if not i.rsid in m:
            m[i.rsid] = (i.non_effect_allele, i.effect_allele, {})
        m[i.rsid][2][i.gene] = i.weight
    return m, weights, extra
    def test_optimized_model_manager(self):
        model_manager = PredictionModel.load_model_manager(
            "tests/_td/dbs_2", Klass=PredictionModel._ModelManager)
        weights = get_weights_in_models("tests/_td/dbs_2")

        #
        self.assertEqual(
            model_manager.get_genes(),
            {'A', 'B', 'C', 'D', 'E', 'F', 'G', 'H', 'I', 'J', 'K'})
        self.assertEqual(model_manager.get_rsids(), set(weights.rsid))
        self.assertEqual(model_manager.get_model_labels(),
                         {"model_sim_1", "model_sim_2"})
        #
        for gene in set(weights.gene):
            w = weights[weights.gene == gene]
            m = model_manager.get_models(gene)
            _compare_o(self, m, w)

        #
        for gene in set(weights.gene):
            self.assertEqual(model_manager.get_model_labels(gene),
                             set(weights[weights.gene == gene].model))
    def test_load_model(self):
        snp_model = PredictionModel.load_model("tests/_td/dbs/test_1.db")

        e_e = SampleData.dataframe_from_extra(SampleData.sample_extra_2())
        numpy.testing.assert_array_equal(
            snp_model.extra[PredictionModel.WDBEQF.K_GENE],
            e_e[PredictionModel.WDBEQF.K_GENE])
        numpy.testing.assert_array_equal(
            snp_model.extra[PredictionModel.WDBEQF.K_GENE_NAME],
            e_e[PredictionModel.WDBEQF.K_GENE_NAME])
        numpy.testing.assert_array_equal(
            snp_model.extra[PredictionModel.WDBEQF.K_N_SNP_IN_MODEL],
            e_e[PredictionModel.WDBEQF.K_N_SNP_IN_MODEL])
        numpy.testing.assert_array_equal(
            snp_model.extra[PredictionModel.WDBEQF.K_PRED_PERF_R2],
            e_e[PredictionModel.WDBEQF.K_PRED_PERF_R2])
        numpy.testing.assert_array_equal(
            snp_model.extra[PredictionModel.WDBEQF.K_PRED_PERF_PVAL],
            e_e[PredictionModel.WDBEQF.K_PRED_PERF_PVAL])
        numpy.testing.assert_array_equal(
            snp_model.extra[PredictionModel.WDBEQF.K_PRED_PERF_QVAL],
            e_e[PredictionModel.WDBEQF.K_PRED_PERF_QVAL])

        e_w = SampleData.dataframe_from_weights(SampleData.sample_weights_2())
        numpy.testing.assert_array_equal(
            snp_model.weights[PredictionModel.WDBQF.K_RSID],
            e_w[PredictionModel.WDBQF.K_RSID])
        numpy.testing.assert_array_equal(
            snp_model.weights[PredictionModel.WDBQF.K_GENE],
            e_w[PredictionModel.WDBQF.K_GENE])
        numpy.testing.assert_array_equal(
            snp_model.weights[PredictionModel.WDBQF.K_WEIGHT],
            e_w[PredictionModel.WDBQF.K_WEIGHT])
        numpy.testing.assert_array_equal(
            snp_model.weights[PredictionModel.WDBQF.K_NON_EFFECT_ALLELE],
            e_w[PredictionModel.WDBQF.K_NON_EFFECT_ALLELE])
        numpy.testing.assert_array_equal(
            snp_model.weights[PredictionModel.WDBQF.K_EFFECT_ALLELE],
            e_w[PredictionModel.WDBQF.K_EFFECT_ALLELE])
示例#12
0
def run(args):
    if os.path.exists(args.snp_covariance_output):
        logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output)
        return

    start = timer()

    logging.info("Loading models...")
    model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern)
    all_snps = model_manager.get_rsids()

    logging.info("processing genotype")
    for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps):
        logging.log(9, "Processing chromosome %s", str(chromosome))
        covariance_results = pandas.DataFrame()

        context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager)
        genes = context.get_genes()
        reporter = Utilities.PercentReporter(9, len(genes))
        reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome))
        for i,gene in enumerate(genes):
            logging.log(6, "%d/%d:%s", i+1, len(genes), gene)
            cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene)
            cov_data = MatrixManager._flatten_matrix_data([cov_data])
            cov_data = Utilities.to_dataframe(cov_data, GenotypeAnalysis.COVARIANCE_COLUMNS, to_numeric="ignore", fill_na="NA")
            covariance_results = pandas.concat([covariance_results, cov_data])

            reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome))

        reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome))

        logging.log(9, "writing chromosome results")
        Utilities.save_dataframe(covariance_results, args.snp_covariance_output,
                                    mode="w" if chromosome ==1 else "a",
                                    header=chromosome==1)

    end = timer()
    logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
示例#13
0
def run(args):
    if os.path.exists(args.snp_covariance_output):
        logging.info("%s already exists, you have to move it or delete it if you want it done again", args.snp_covariance_output)
        return

    start = timer()

    logging.info("Loading models...")
    model_manager = PredictionModel.load_model_manager(args.models_folder, name_pattern=args.models_pattern, name_filter=args.models_filter)
    all_snps = model_manager.get_rsids()
    Utilities.ensure_requisite_folders(args.snp_covariance_output)
    with gzip.open(args.snp_covariance_output, "w") as o:
        o.write("GENE\tRSID1\tRSID2\tVALUE\n")
        logging.info("processing genotype")

        for chromosome, metadata, dosage in GenotypeUtilities.genotype_by_chromosome_from_args(args, all_snps):
            logging.log(9, "Processing chromosome %s", str(chromosome))

            context = GenotypeAnalysis.GenotypeAnalysisContext(metadata, dosage, model_manager)
            genes = context.get_genes()
            reporter = Utilities.PercentReporter(9, len(genes))
            reporter.update(0, "%d %% of genes processed so far in chromosome " + str(chromosome))
            for i,gene in enumerate(genes):
                logging.log(6, "%d/%d:%s", i+1, len(genes), gene)
                cov_data = GenotypeAnalysis.get_prediction_covariance(context, gene)
                cov_data = MatrixManager._flatten_matrix_data([cov_data])
                for e in cov_data:
                    l = "{}\t{}\t{}\t{}\n".format(e[0], e[1], e[2], e[3])
                    o.write(l)

                reporter.update(i, "%d %% of genes processed so far in chromosome "+str(chromosome))

            reporter.update(len(genes), "%d %% of genes processed so far in chromosome " + str(chromosome))

    end = timer()
    logging.info("Ran covariance builder in %s seconds" % (str(end - start)))
def _prediction_model():
    e = SampleData.dataframe_from_extra(SampleData.sample_extra_1())
    w = SampleData.dataframe_from_weights(SampleData.sample_weights_1())
    p = PredictionModel.Model(w, e)
    return p
 def test_snps_in_db(self):
     expected = {"rs245915", "rs245913", "rs245909", "rs245906", "rs10486599", "rs144012121", "rs117887801", "rs542000", "rs544632",
                 "rs498475", "rs849327", "rs849336", "rs849335", "rs1513272", "rs849135", "rs849134", "rs860262", "rs849133", "rs1635852",
                 "rs864745", "rs112751321", "rs144273091", "rs117462481", "rs149305679", "rs643036", "rs1937888", "rs17155745", "rs62626328"}
     actual = PredictionModel.snps_in_db("tests/_td/dbs/test_2.db")
     self.assertEqual(actual, expected)