Exemple #1
0
def consensus_run(args, networks, heats, verbose):
    # Perform the single runs
    single_runs = []
    for (infmat, indexToGene, G, nname,
         pnp), (heat, hname) in product(networks, heats):
        # Simple progress bar
        if args.verbose > 0: print '\t-', nname, hname

        # 1) Filter the heat scores
        # 1a) Remove enes not in the network
        heat = filter_heat_to_network_genes(heat, set(indexToGene.values()),
                                            verbose)

        # 1b) Genes with score 0 cannot be in output components, but are eligible for heat in permutations
        heat, addtl_genes = filter_heat(
            heat, None, False, 'There are ## genes with heat score 0')

        if args.verbose > 1:
            print "\t\t- Loaded '%s' heat scores for %s genes" % (hname,
                                                                  len(heat))

        result = run_helper(args, infmat, indexToGene, G, nname, pnp, heat,
                            hname, addtl_genes, get_deltas_hotnet2,
                            HN2_INFMAT_NAME, HN2_MAX_CC_SIZES, args.verbose)
        single_runs.append((nname, hname, result))

    # Perform the consensus
    consensus, linkers, auto_deltas = identify_consensus(single_runs,
                                                         verbose=verbose)

    return single_runs, consensus, linkers, auto_deltas
Exemple #2
0
def consensus_run(args, networks, heats, verbose):
    # Perform the single runs
    single_runs = []
    """Change from here"""
    #single_permuted_sub_score_delta = []
    #single_subnet_score_delta = []
    #single_Alpha_sig_delta = []
    #signle_Subnet_sig_delta = []
    #single_Conp_sig_delta = []
    #single_sig_Count_delta = []
    #single_sig_Conp_delta = []
    #single_degree_delta = []
    #single_permuted_degree_delta = []
    #single_cent_weighted_score_delta = []
    #single_degree_weighted_score_delta = []
    single_My_results = []
    """end here"""
    for (infmat, indexToGene, G, nname,
         pnp), (heat, hname) in product(networks, heats):
        # Simple progress bar
        if args.verbose > 0: print '\t-', nname, hname

        # 1) Filter the heat scores
        # 1a) Remove enes not in the network
        heat = filter_heat_to_network_genes(heat, set(indexToGene.values()),
                                            verbose)

        # 1b) Genes with score 0 cannot be in output components, but are eligible for heat in permutations
        heat, addtl_genes = filter_heat(
            heat, None, False, 'There are ## genes with heat score 0')

        if args.verbose > 1:
            print "\t\t- Loaded '%s' heat scores for %s genes" % (hname,
                                                                  len(heat))

        result, My_results = run_helper(args, infmat, indexToGene, G, nname,
                                        pnp, heat, hname, addtl_genes,
                                        get_deltas_hotnet2, HN2_INFMAT_NAME,
                                        HN2_MAX_CC_SIZES, args.verbose)

        single_runs.append((nname, hname, result))
        single_My_results.append((nname, hname, My_results))
        """Change from here"""
        #single_permuted_sub_score_delta.append((nname, hname,permuted_sub_score_delta))
        #single_subnet_score_delta.append((nname, hname,subnet_geneScore_delta))
        #single_Alpha_sig_delta.append((nname, hname,Alpha_sig_delta))
        #signle_Subnet_sig_delta.append((nname, hname,Subnet_sig_delta))
        #single_Conp_sig_delta.append((nname, hname,Conp_sig_delta))
        #single_sig_Conp_delta.append((nname, hname,sig_Conp_delta))
        #single_sig_Count_delta.append((nname, hname,sig_Count_delta))
        #single_degree_delta.append((nname,hname,degree_delta))
        #single_permuted_degree_delta.append((nname,hname,permuted_degree_delta))
        #single_cent_weighted_score_delta.append((nname,hname,cent_weighted_score_delta))
        #single_degree_weighted_score_delta.append((nname,hname,degree_weighted_score_delta))
        """End here"""
    # Perform the consensus
    consensus, linkers, auto_deltas = identify_consensus(single_runs,
                                                         verbose=verbose)
    #print single_degree_delta
    return single_runs, single_My_results, consensus, linkers, auto_deltas
Exemple #3
0
def consensus_with_stats(args, networks, heats, verbose=0):
    # Run with the input heat
    """Change here, add the single_permuted_sub_score_delta output"""
    single_runs, single_My_results, consensus, linkers, auto_deltas = consensus_run(
        args, networks, heats, verbose)

    # Generate permuted heats
    np = args.consensus_permutations
    permuted_single_runs = defaultdict(list)
    for (infmat, indexToGene, G, nname,
         pnp), (heat, hname) in product(networks, heats):
        # 1) Filter the heat scores
        # 1a) Remove enes not in the network
        heat = filter_heat_to_network_genes(heat, set(indexToGene.values()),
                                            verbose)

        # 1b) Genes with score 0 cannot be in output components, but are eligible for heat in permutations
        heat, addtl_genes = filter_heat(
            heat, None, False, 'There are ## genes with heat score 0')

        for permutation in permute_heat(heat, indexToGene.values(), np,
                                        addtl_genes, args.num_cores):
            result, Myresult = run_helper(args,
                                          infmat,
                                          indexToGene,
                                          G,
                                          nname,
                                          pnp,
                                          heat,
                                          hname,
                                          addtl_genes,
                                          get_deltas_hotnet2,
                                          HN2_INFMAT_NAME,
                                          HN2_MAX_CC_SIZES,
                                          verbose=verbose)
            permuted_single_runs[(hname, nname)].append(result)

    # Run consensus to compute observed statistics
    network_heat_pairs = permuted_single_runs.keys()
    permuted_counts = []
    for i in range(args.heat_permutations):
        runs = [(n, h, permuted_single_runs[(n, h)][i])
                for n, h in network_heat_pairs]
        permuted_consensus, _, _ = identify_consensus(runs, verbose=verbose)
        permuted_counts.append(count_consensus(permuted_consensus))

    # Summarize stats
    consensus_stats = dict()
    for k, count in count_consensus(consensus).iteritems():
        empirical = [permuted_count[k] for permuted_count in permuted_counts]
        if np == 0:
            pval = 1.
            expected = 0.
        else:
            expected = numpy.mean(empirical)
            pval = sum(1. for p in empirical if p >= count) / np
        consensus_stats[k] = dict(observed=count, expected=expected, pval=pval)

    return single_runs, single_My_results, consensus, linkers, auto_deltas, consensus_stats
Exemple #4
0
def load_direct_heat(args):
    heat = hnio.load_heat_tsv(args.heat_file)
    heat, score_excluded_genes, args.min_heat_score = hnheat.filter_heat(heat, args.min_heat_score)
    
    filter_excluded_genes = []
    if args.gene_filter_file:
        heat, filter_excluded_genes = hnheat.expr_filter_heat(heat,
                                                              hnio.load_genes(args.gene_filter_file))
    
    #ensure that all heat scores are positive
    bad_genes = [gene for gene in heat if heat[gene] < 0]
    if bad_genes:
        raise ValueError("ERROR: All gene heat scores must be non-negative. There are %s genes with\
                          negative heat scores: %s" % (len(bad_genes), bad_genes))
    
    return heat, list(set(score_excluded_genes + filter_excluded_genes))
Exemple #5
0
def load_direct_heat(args):
    heat = hnio.load_heat_tsv(args.heat_file)
    heat, score_excluded_genes, args.min_heat_score = hnheat.filter_heat(
        heat, args.min_heat_score)

    filter_excluded_genes = []
    if args.gene_filter_file:
        heat, filter_excluded_genes = hnheat.expr_filter_heat(
            heat, hnio.load_genes(args.gene_filter_file))

    #ensure that all heat scores are positive
    bad_genes = [gene for gene in heat if heat[gene] < 0]
    if bad_genes:
        raise ValueError(
            "ERROR: All gene heat scores must be non-negative. There are %s genes with\
                          negative heat scores: %s" %
            (len(bad_genes), bad_genes))

    return heat, list(set(score_excluded_genes + filter_excluded_genes))
Exemple #6
0
def consensus_with_stats(args, networks, heats, verbose=0):
    # Run with the input heat
    single_runs, consensus, linkers, auto_deltas = consensus_run( args, networks, heats, verbose )

    # Generate permuted heats
    np = args.consensus_permutations
    permuted_single_runs = defaultdict(list)
    for (infmat, indexToGene, G, nname, pnp), (heat, hname) in product(networks, heats):
        # 1) Filter the heat scores
        # 1a) Remove genes not in the network
        heat = filter_heat_to_network_genes(heat, set(indexToGene.values()), verbose)

        # 1b) Genes with score 0 cannot be in output components, but are eligible for heat in permutations
        heat, addtl_genes = filter_heat(heat, None, False, 'There are ## genes with heat score 0')

        for permutation in permute_heat(heat, indexToGene.values(), np, addtl_genes, args.num_cores):
            result = run_helper(args, infmat, indexToGene, G, nname, pnp, heat, hname, addtl_genes, get_deltas_hotnet2, HN2_INFMAT_NAME, HN2_MAX_CC_SIZES, verbose=verbose)
            permuted_single_runs[(hname, nname)].append(result)

    # Run consensus to compute observed statistics
    network_heat_pairs = permuted_single_runs.keys()
    permuted_counts = []
    for i in range(np):
        runs = [ (n, h, permuted_single_runs[(n, h)][i]) for n, h in network_heat_pairs ]
        permuted_consensus, _, _ = identify_consensus( runs, verbose=verbose )
        permuted_counts.append(count_consensus(permuted_consensus))

    # Summarize stats
    consensus_stats = dict()
    for k, count in count_consensus(consensus).iteritems():
        empirical = [ permuted_count[k] for permuted_count in permuted_counts ]
        if np == 0:
            pval     = 1.
            expected = 0.
        else:
            expected = numpy.mean(empirical)
            pval     = sum(1. for p in empirical if p >= count )/np
        consensus_stats[k] = dict(observed=count, expected=expected, pval=pval)

    return single_runs, consensus, linkers, auto_deltas, consensus_stats
Exemple #7
0
def consensus_run(args, networks, heats, verbose):
    # Perform the single runs
    single_runs = []
    for (infmat, indexToGene, G, nname, pnp), (heat, hname) in product(networks, heats):
        # Simple progress bar
        if args.verbose > 0: print '\t-', nname, hname

        # 1) Filter the heat scores
        # 1a) Remove enes not in the network
        heat = filter_heat_to_network_genes(heat, set(indexToGene.values()), verbose)

        # 1b) Genes with score 0 cannot be in output components, but are eligible for heat in permutations
        heat, addtl_genes = filter_heat(heat, None, False, 'There are ## genes with heat score 0')

        if args.verbose > 1:
            print "\t\t- Loaded '%s' heat scores for %s genes" % (hname, len(heat))

        result = run_helper(args, infmat, indexToGene, G, nname, pnp, heat, hname, addtl_genes, get_deltas_hotnet2, HN2_INFMAT_NAME, HN2_MAX_CC_SIZES, args.verbose)
        single_runs.append( (nname, hname, result) )

    # Perform the consensus
    consensus, linkers, auto_deltas = identify_consensus( single_runs, verbose=verbose )

    return single_runs, consensus, linkers, auto_deltas
Exemple #8
0
def run_helper(args, infmat_name, get_deltas_fn, extra_delta_args):
    """Helper shared by simpleRun and simpleRunClassic.
    """
    # create output directory if doesn't exist; warn if it exists and is not empty
    if not os.path.exists(args.output_directory):
        os.makedirs(args.output_directory)
    if len(os.listdir(args.output_directory)) > 0:
        print("WARNING: Output directory is not empty. Any conflicting files will be overwritten. "
              "(Ctrl-c to cancel).")

    infmat = hnio.load_infmat(args.infmat_file, infmat_name)
    full_index2gene = hnio.load_index(args.infmat_index_file)
    
    using_json_heat = os.path.splitext(args.heat_file.lower())[1] == '.json'
    if using_json_heat:
        heat = json.load(open(args.heat_file))['heat']
    else:
        heat = hnio.load_heat_tsv(args.heat_file)
    print "* Loaded heat scores for %s genes" % len(heat)
    
    # filter out genes not in the network
    heat = hnheat.filter_heat_to_network_genes(heat, set(full_index2gene.values()))
    
    # genes with score 0 cannot be in output components, but are eligible for heat in permutations
    heat, addtl_genes = hnheat.filter_heat(heat, None, False, 'There are ## genes with heat score 0')
    
    deltas = get_deltas_fn(full_index2gene, heat, args.delta_permutations, args.num_cores, infmat, addtl_genes, *extra_delta_args)
    
    sim, index2gene = hn.similarity_matrix(infmat, full_index2gene, heat, True)

    results_files = []
    for delta in deltas:
        # create output directory
        delta_out_dir = args.output_directory + "/delta_" + str(delta)
        if not os.path.isdir(delta_out_dir):
            os.mkdir(delta_out_dir)
        
        # find connected components
        G = hn.weighted_graph(sim, index2gene, delta, directed=True)
        ccs = hn.connected_components(G, args.min_cc_size)
        
        # calculate significance (using all genes with heat scores)
        print "* Performing permuted heat statistical significance..."
        heat_permutations = p.permute_heat(heat, full_index2gene.values(),
                                           args.significance_permutations, addtl_genes,
                                           args.num_cores)
        sizes = range(2, 11)
        print "\t- Using no. of components >= k (k \\in",
        print "[%s, %s]) as statistic" % (min(sizes), max(sizes))
        sizes2counts = stats.calculate_permuted_cc_counts(infmat, full_index2gene,
                                                          heat_permutations, delta, sizes, True,
                                                          args.num_cores)
        real_counts = stats.num_components_min_size(G, sizes)
        size2real_counts = dict(zip(sizes, real_counts))
        sizes2stats = stats.compute_statistics(size2real_counts, sizes2counts,
                                               args.significance_permutations)
    
        # sort ccs list such that genes within components are sorted alphanumerically, and components
        # are sorted first by length, then alphanumerically by name of the first gene in the component
        ccs = [sorted(cc) for cc in ccs]
        ccs.sort(key=lambda comp: comp[0])
        ccs.sort(key=len, reverse=True)

        # write output
        if not using_json_heat:
            heat_dict = {"heat": heat, "parameters": {"heat_file": args.heat_file}}
            heat_out = open(os.path.abspath(delta_out_dir) + "/" + HEAT_JSON, 'w')
            json.dump(heat_dict, heat_out, indent=4)
            heat_out.close()
            args.heat_file = os.path.abspath(delta_out_dir) + "/" + HEAT_JSON
        
        args.delta = delta  # include delta in parameters section of output JSON
        output_dict = {"parameters": vars(args), "sizes": hn.component_sizes(ccs),
                       "components": ccs, "statistics": sizes2stats}
        hnio.write_significance_as_tsv(os.path.abspath(delta_out_dir) + "/" + SIGNIFICANCE_TSV,
                                       sizes2stats)
        
        json_out = open(os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT, 'w')
        json.dump(output_dict, json_out, indent=4)
        json_out.close()
        results_files.append( os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT )
        
        hnio.write_components_as_tsv(os.path.abspath(delta_out_dir) + "/" + COMPONENTS_TSV, ccs)

    # create the hierarchy if necessary
    if args.output_hierarchy:
        from bin import createDendrogram as CD

        hierarchy_out_dir = '{}/hierarchy/'.format(args.output_directory)
        if not os.path.isdir(hierarchy_out_dir): os.mkdir(hierarchy_out_dir)

        params = vars(args)
        CD.createDendrogram( sim, index2gene.values(), hierarchy_out_dir, params, verbose=False)
        hierarchyFile = '{}/viz_files/{}'.format(str(hn.__file__).rsplit('/', 1)[0], HIERARCHY_WEB_FILE)
        shutil.copy(hierarchyFile, '{}/index.html'.format(hierarchy_out_dir))

    # write visualization output if edge file given
    if args.edge_file:
        from bin import makeResultsWebsite as MRW
        viz_args = [ "-r" ] + results_files
        viz_args += ["-ef", args.edge_file, "-o", args.output_directory + "/viz" ]
        if args.network_name: viz_args += [ "-nn", args.network_name ]
        if args.display_score_file: viz_args += [ "-dsf", args.display_score_file ]
        if args.display_name_file: viz_args += [ "-dnf", args.display_name_file ]
        MRW.run( MRW.get_parser().parse_args(viz_args) )
Exemple #9
0
def run_helper(args, infmat_name, get_deltas_fn, extra_delta_args):
    """Helper shared by simpleRun and simpleRunClassic.
    """
    # create output directory if doesn't exist; warn if it exists and is not empty
    if not os.path.exists(args.output_directory):
        os.makedirs(args.output_directory)
    if len(os.listdir(args.output_directory)) > 0:
        print("WARNING: Output directory is not empty. Any conflicting files will be overwritten. "
              "(Ctrl-c to cancel).")
    
    infmat = scipy.io.loadmat(args.infmat_file)[infmat_name]
    full_index2gene = hnio.load_index(args.infmat_index_file)
    
    using_json_heat = os.path.splitext(args.heat_file.lower())[1] == '.json'
    if using_json_heat:
        heat = json.load(open(args.heat_file))['heat']
    else:
        heat = hnio.load_heat_tsv(args.heat_file)
    print "* Loaded heat scores for %s genes" % len(heat)
    
    # filter out genes not in the network
    heat = hnheat.filter_heat_to_network_genes(heat, set(full_index2gene.values()))
    
    # genes with score 0 cannot be in output components, but are eligible for heat in permutations
    heat, addtl_genes = hnheat.filter_heat(heat, None, False, 'There are ## genes with heat score 0')
    
    deltas = get_deltas_fn(full_index2gene, heat, args.delta_permutations, args.num_cores, infmat, addtl_genes, *extra_delta_args)
    
    sim, index2gene = hn.similarity_matrix(infmat, full_index2gene, heat, True)

    results_files = []
    for delta in deltas:
        # create output directory
        delta_out_dir = args.output_directory + "/delta_" + str(delta)
        if not os.path.isdir(delta_out_dir):
            os.mkdir(delta_out_dir)
        
        # find connected components
        G = hn.weighted_graph(sim, index2gene, delta, directed=True)
        ccs = hn.connected_components(G, args.min_cc_size)
        
        # calculate significance (using all genes with heat scores)
        print "* Performing permuted heat statistical significance..."
        heat_permutations = p.permute_heat(heat, full_index2gene.values(),
                                           args.significance_permutations, addtl_genes,
                                           args.num_cores)
        sizes = range(2, 11)
        print "\t- Using no. of components >= k (k \\in",
        print "[%s, %s]) as statistic" % (min(sizes), max(sizes))
        sizes2counts = stats.calculate_permuted_cc_counts(infmat, full_index2gene,
                                                          heat_permutations, delta, sizes, True,
                                                          args.num_cores)
        real_counts = stats.num_components_min_size(G, sizes)
        size2real_counts = dict(zip(sizes, real_counts))
        sizes2stats = stats.compute_statistics(size2real_counts, sizes2counts,
                                               args.significance_permutations)
    
        # sort ccs list such that genes within components are sorted alphanumerically, and components
        # are sorted first by length, then alphanumerically by name of the first gene in the component
        ccs = [sorted(cc) for cc in ccs]
        ccs.sort(key=lambda comp: comp[0])
        ccs.sort(key=len, reverse=True)

        # write output
        if not using_json_heat:
            heat_dict = {"heat": heat, "parameters": {"heat_file": args.heat_file}}
            heat_out = open(os.path.abspath(delta_out_dir) + "/" + HEAT_JSON, 'w')
            json.dump(heat_dict, heat_out, indent=4)
            heat_out.close()
            args.heat_file = os.path.abspath(delta_out_dir) + "/" + HEAT_JSON
        
        args.delta = delta  # include delta in parameters section of output JSON
        output_dict = {"parameters": vars(args), "sizes": hn.component_sizes(ccs),
                       "components": ccs, "statistics": sizes2stats}
        hnio.write_significance_as_tsv(os.path.abspath(delta_out_dir) + "/" + SIGNIFICANCE_TSV,
                                       sizes2stats)
        
        json_out = open(os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT, 'w')
        json.dump(output_dict, json_out, indent=4)
        json_out.close()
        results_files.append( os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT )
        
        hnio.write_components_as_tsv(os.path.abspath(delta_out_dir) + "/" + COMPONENTS_TSV, ccs)

    # write visualization output if edge file given
    if args.edge_file:
        from bin import makeResultsWebsite as MRW
        viz_args = [ "-r" ] + results_files
        viz_args += ["-ef", args.edge_file, "-o", args.output_directory + "/viz" ]
        if args.network_name: viz_args += [ "-nn", args.network_name ]
        if args.display_score_file: viz_args += [ "-dsf", args.display_score_file ]
        MRW.run( MRW.get_parser().parse_args(viz_args) )
Exemple #10
0
def run(args):
    # create output directory if doesn't exist; warn if it exists and is not empty
    if not os.path.exists(args.output_directory):
        os.makedirs(args.output_directory)
    if len(os.listdir(args.output_directory)) > 0:
        print("WARNING: Output directory is not empty. Any conflicting files will be overwritten. ")
        print("(Ctrl-c to cancel).")
    
    infmat = scipy.io.loadmat(args.infmat_file)[INFMAT_NAME]
    infmat_index = hnio.load_index(args.infmat_index_file)
    heat = hnio.load_heat_tsv(args.heat_file)
    
    # filter out genes with heat score less than min_heat_score
    heat, addtl_genes, args.min_heat_score = hnheat.filter_heat(heat, args.min_heat_score)
    
    # find delta that maximizes # CCs of size >= MIN_SIZE for each permuted data set
    deltas = ft.get_deltas_for_heat(infmat, infmat_index, heat, addtl_genes, args.num_permutations,
                                    args.parallel)

    #find the multiple of the median delta s.t. the size of the largest CC in the real data
    #is <= MAX_CC_SIZE
    medianDelta = np.median(deltas[MIN_CC_SIZE])
    M, gene_index = hn.induce_infmat(infmat, infmat_index, sorted(heat.keys()), quiet=False)
    h = hn.heat_vec(heat, gene_index)
    sim = hn.similarity_matrix(M, h)
    
    for i in range(1, 11):
        G = hn.weighted_graph(sim, gene_index, i*medianDelta)
        max_cc_size = max([len(cc) for cc in hn.connected_components(G)])
        if max_cc_size <= MAX_CC_SIZE:
            break
    
    # load interaction network edges and determine location of static HTML files for visualization
    edges = hnio.load_ppi_edges(args.edge_file) if args.edge_file else None
    index_file = '%s/viz_files/%s' % (os.path.realpath(__file__).rsplit('/', 1)[0], VIZ_INDEX)
    subnetworks_file = '%s/viz_files/%s' % (os.path.realpath(__file__).rsplit('/', 1)[0], VIZ_SUBNETWORKS)
    gene2index = dict([(gene, index) for index, gene in list(infmat_index.items())])

    #and run HotNet with that multiple and the next 4 multiples
    run_deltas = [i*medianDelta for i in range(i, i+5)]
    for delta in run_deltas: 
        # create output directory
        delta_out_dir = args.output_directory + "/delta_" + str(delta)
        if not os.path.isdir(delta_out_dir):
            os.mkdir(delta_out_dir)
        
        # find connected components
        G = hn.weighted_graph(sim, gene_index, delta)
        ccs = hn.connected_components(G, args.min_cc_size)
        
        # calculate significance (using all genes with heat scores)
        print("* Performing permuted heat statistical significance...")
        heat_permutations = p.permute_heat(heat, args.num_permutations, addtl_genes, args.parallel)
        sizes = list(range(2, 11))
        print("\t- Using no. of components >= k (k \\in")
        print("[%s, %s]) as statistic" % (min(sizes), max(sizes)))
        sizes2counts = stats.calculate_permuted_cc_counts(infmat, infmat_index, heat_permutations,
                                                          delta, sizes, args.parallel)
        real_counts = stats.num_components_min_size(G, sizes)
        size2real_counts = dict(list(zip(sizes, real_counts)))
        sizes2stats = stats.compute_statistics(size2real_counts, sizes2counts, args.num_permutations)
    
        # sort ccs list such that genes within components are sorted alphanumerically, and components
        # are sorted first by length, then alphanumerically by name of the first gene in the component 
        ccs = [sorted(cc) for cc in ccs]
        ccs.sort(key=lambda comp: comp[0])
        ccs.sort(key=len, reverse=True)
    
        # write output
        heat_dict = {"heat": heat, "parameters": {"heat_file": args.heat_file}}
        heat_out = open(os.path.abspath(delta_out_dir) + "/" + HEAT_JSON, 'w')
        json.dump(heat_dict, heat_out, indent=4)
        heat_out.close()
        
        args.heat_file = os.path.abspath(delta_out_dir) + "/" + HEAT_JSON
        args.delta = delta
        output_dict = {"parameters": vars(args), "sizes": hn.component_sizes(ccs),
                       "components": ccs, "statistics": sizes2stats}
        hnio.write_significance_as_tsv(os.path.abspath(delta_out_dir) + "/" + SIGNIFICANCE_TSV,
                                       sizes2stats)
        
        json_out = open(os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT, 'w')
        json.dump(output_dict, json_out, indent=4)
        json_out.close()
        
        hnio.write_components_as_tsv(os.path.abspath(delta_out_dir) + "/" + COMPONENTS_TSV, ccs)

        # write visualization output if edge file given
        if args.edge_file:
            viz_data = {"delta": delta, 'subnetworks': list()}
            for cc in ccs:
                viz_data['subnetworks'].append(viz.get_component_json(cc, heat, edges, gene2index, args.network_name))
                
            delta_viz_dir = '%s/viz/delta%s' % (args.output_directory, delta)
            if not os.path.isdir(delta_viz_dir):
                os.makedirs(delta_viz_dir)
            viz_out = open('%s/subnetworks.json' % delta_viz_dir, 'w')
            json.dump(viz_data, viz_out, indent=4)
            viz_out.close()
   
            shutil.copy(subnetworks_file, delta_viz_dir)
    
    if args.edge_file:
        viz.write_index_file(index_file, '%s/viz/%s' % (args.output_directory, VIZ_INDEX), run_deltas)
Exemple #11
0
def run(args):
    # create output directory if doesn't exist; warn if it exists and is not empty
    if not os.path.exists(args.output_directory):
        os.makedirs(args.output_directory)
    if len(os.listdir(args.output_directory)) > 0:
        print("WARNING: Output directory is not empty. Any conflicting files will be overwritten. "
              "(Ctrl-c to cancel).")
    
    infmat = scipy.io.loadmat(args.infmat_file)[INFMAT_NAME]
    infmat_index = hnio.load_index(args.infmat_index_file)
    heat = hnio.load_heat_tsv(args.heat_file)
    
    #filter out genes with heat score less than min_heat_score
    heat, addtl_genes, args.min_heat_score = hnheat.filter_heat(heat, args.min_heat_score)
    
    #find delta that maximizes # CCs of size >= MIN_SIZE for each permuted data set
    deltas = ft.get_deltas_for_heat(infmat, infmat_index, heat, addtl_genes, args.num_permutations,
                                    args.parallel)

    #find the multiple of the median delta s.t. the size of the largest CC in the real data
    #is <= MAX_CC_SIZE
    medianDelta = np.median(deltas[MIN_CC_SIZE])
    M, gene_index = hn.induce_infmat(infmat, infmat_index, sorted(heat.keys()), quiet=False)
    h = hn.heat_vec(heat, gene_index)
    sim = hn.similarity_matrix(M, h)
    
    for i in range(1, 11):
        G = hn.weighted_graph(sim, gene_index, i*medianDelta)
        max_cc_size = max([len(cc) for cc in hn.connected_components(G)])
        if max_cc_size <= MAX_CC_SIZE:
            break
    
    #and run HotNet with that multiple and the next 4 multiples
    run_deltas = [i*medianDelta for i in range(i, i+5)]
    for delta in run_deltas: 
        #create output directory
        delta_out_dir = args.output_directory + "/delta_" + str(delta)
        if not os.path.isdir(delta_out_dir):
            os.mkdir(delta_out_dir)
        
        #find connected components
        G = hn.weighted_graph(sim, gene_index, delta)
        ccs = hn.connected_components(G, args.min_cc_size)
        
        # calculate significance (using all genes with heat scores)
        print "* Performing permuted heat statistical significance..."
        heat_permutations = p.permute_heat(heat, args.num_permutations, addtl_genes, args.parallel)
        sizes = range(2, 11)
        print "\t- Using no. of components >= k (k \\in",
        print "[%s, %s]) as statistic" % (min(sizes), max(sizes))
        sizes2counts = stats.calculate_permuted_cc_counts(infmat, infmat_index, heat_permutations,
                                                          delta, sizes, args.parallel)
        real_counts = stats.num_components_min_size(G, sizes)
        size2real_counts = dict(zip(sizes, real_counts))
        sizes2stats = stats.compute_statistics(size2real_counts, sizes2counts, args.num_permutations)
    
        #sort ccs list such that genes within components are sorted alphanumerically, and components
        #are sorted first by length, then alphanumerically by name of the first gene in the component 
        ccs = [sorted(cc) for cc in ccs]
        ccs.sort(key=lambda comp: comp[0])
        ccs.sort(key=len, reverse=True)
    
        # write output
        output_dict = {"parameters": vars(args), "sizes": hn.component_sizes(ccs),
                       "components": ccs, "statistics": sizes2stats}
        hnio.write_significance_as_tsv(os.path.abspath(delta_out_dir) + "/" + SIGNIFICANCE_TSV,
                                       sizes2stats)
        
        json_out = open(os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT, 'w')
        json.dump(output_dict, json_out, indent=4)
        json_out.close()
        
        hnio.write_components_as_tsv(os.path.abspath(delta_out_dir) + "/" + COMPONENTS_TSV, ccs)
Exemple #12
0
def run(args):
    # create output directory if doesn't exist; warn if it exists and is not empty
    if not os.path.exists(args.output_directory):
        os.makedirs(args.output_directory)
    if len(os.listdir(args.output_directory)) > 0:
        print("WARNING: Output directory is not empty. Any conflicting files will be overwritten. "
              "(Ctrl-c to cancel).")
    
    infmat = scipy.io.loadmat(args.infmat_file)[INFMAT_NAME]
    infmat_index = hnio.load_index(args.infmat_index_file)
    heat = hnio.load_heat_tsv(args.heat_file)
    
    # filter out genes with heat score less than min_heat_score
    heat, addtl_genes, args.min_heat_score = hnheat.filter_heat(heat, args.min_heat_score)
    
    # find delta that maximizes # CCs of size >= MIN_SIZE for each permuted data set
    deltas = ft.get_deltas_for_heat(infmat, infmat_index, heat, addtl_genes, args.num_permutations,
                                    args.parallel)

    #find the multiple of the median delta s.t. the size of the largest CC in the real data
    #is <= MAX_CC_SIZE
    medianDelta = np.median(deltas[MIN_CC_SIZE])
    M, gene_index = hn.induce_infmat(infmat, infmat_index, sorted(heat.keys()), quiet=False)
    h = hn.heat_vec(heat, gene_index)
    sim = hn.similarity_matrix(M, h)
    
    for i in range(1, 11):
        G = hn.weighted_graph(sim, gene_index, i*medianDelta)
        max_cc_size = max([len(cc) for cc in hn.connected_components(G)])
        if max_cc_size <= MAX_CC_SIZE:
            break
    
    # load interaction network edges and determine location of static HTML files for visualization
    edges = hnio.load_ppi_edges(args.edge_file) if args.edge_file else None
    index_file = '%s/viz_files/%s' % (os.path.realpath(__file__).rsplit('/', 1)[0], VIZ_INDEX)
    subnetworks_file = '%s/viz_files/%s' % (os.path.realpath(__file__).rsplit('/', 1)[0], VIZ_SUBNETWORKS)
    gene2index = dict([(gene, index) for index, gene in infmat_index.iteritems()])

    #and run HotNet with that multiple and the next 4 multiples
    run_deltas = [i*medianDelta for i in range(i, i+5)]
    for delta in run_deltas: 
        # create output directory
        delta_out_dir = args.output_directory + "/delta_" + str(delta)
        if not os.path.isdir(delta_out_dir):
            os.mkdir(delta_out_dir)
        
        # find connected components
        G = hn.weighted_graph(sim, gene_index, delta)
        ccs = hn.connected_components(G, args.min_cc_size)
        
        # calculate significance (using all genes with heat scores)
        print "* Performing permuted heat statistical significance..."
        heat_permutations = p.permute_heat(heat, args.num_permutations, addtl_genes, args.parallel)
        sizes = range(2, 11)
        print "\t- Using no. of components >= k (k \\in",
        print "[%s, %s]) as statistic" % (min(sizes), max(sizes))
        sizes2counts = stats.calculate_permuted_cc_counts(infmat, infmat_index, heat_permutations,
                                                          delta, sizes, args.parallel)
        real_counts = stats.num_components_min_size(G, sizes)
        size2real_counts = dict(zip(sizes, real_counts))
        sizes2stats = stats.compute_statistics(size2real_counts, sizes2counts, args.num_permutations)
    
        # sort ccs list such that genes within components are sorted alphanumerically, and components
        # are sorted first by length, then alphanumerically by name of the first gene in the component 
        ccs = [sorted(cc) for cc in ccs]
        ccs.sort(key=lambda comp: comp[0])
        ccs.sort(key=len, reverse=True)
    
        # write output
        heat_dict = {"heat": heat, "parameters": {"heat_file": args.heat_file}}
        heat_out = open(os.path.abspath(delta_out_dir) + "/" + HEAT_JSON, 'w')
        json.dump(heat_dict, heat_out, indent=4)
        heat_out.close()
        
        args.heat_file = os.path.abspath(delta_out_dir) + "/" + HEAT_JSON
        args.delta = delta
        output_dict = {"parameters": vars(args), "sizes": hn.component_sizes(ccs),
                       "components": ccs, "statistics": sizes2stats}
        hnio.write_significance_as_tsv(os.path.abspath(delta_out_dir) + "/" + SIGNIFICANCE_TSV,
                                       sizes2stats)
        
        json_out = open(os.path.abspath(delta_out_dir) + "/" + JSON_OUTPUT, 'w')
        json.dump(output_dict, json_out, indent=4)
        json_out.close()
        
        hnio.write_components_as_tsv(os.path.abspath(delta_out_dir) + "/" + COMPONENTS_TSV, ccs)

        # write visualization output if edge file given
        if args.edge_file:
            viz_data = {"delta": delta, 'subnetworks': list()}
            for cc in ccs:
                viz_data['subnetworks'].append(viz.get_component_json(cc, heat, edges, gene2index,
                                                                      args.network_name))
                
            delta_viz_dir = '%s/viz/delta%s' % (args.output_directory, delta)
            if not os.path.isdir(delta_viz_dir):
                os.makedirs(delta_viz_dir)
            viz_out = open('%s/subnetworks.json' % delta_viz_dir, 'w')
            json.dump(viz_data, viz_out, indent=4)
            viz_out.close()
   
            shutil.copy(subnetworks_file, delta_viz_dir)
    
    if args.edge_file:
        viz.write_index_file(index_file, '%s/viz/%s' % (args.output_directory, VIZ_INDEX), run_deltas)