예제 #1
0
def get_ensemble(data_sampled, rule):
    n_clusts = rule(n)

    logger.info("* * * * * * * * * * * * * * * * * *")
    logger.info("Num. samples: {}".format(n))
    logger.info("New config: {}".format(rule.__doc__))
    logger.info("* * * * * * * * * * * * * * * * * *")

    # skip if number of clusters is bigger than number of samples
    if n_clusts[1] >= n:
        logger.info("Kmax too large for dataset size. Skipping...")
        continue
    if n_clusts[0] <= 1:
        logger.info("Kmin too little. Skipping...")
        continue            


    ## generate ensemble
    logger.info("Checking for ensemble in folder...")

    generator = myKM.K_Means(cuda_mem="manual")    

    # if there is an ensemble file load it, otherwise generate and save
    ensemble_filename = os.path.join(folder,"ensemble_{}_{}.hdf".format(n, rule.__doc__))
    if not os.path.exists(ensemble_filename):
        logger.info("No ensemble detected. Generating ensemble...")
        t.reset()
        t.tic()
        ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
                                         n_partitions, n_iters)
        t.tac()
        part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
        logger.info("Saved ensemble in file: {}".format(ensemble_filename))
        t_ensemble = t.elapsed
    else:
        logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename))
        ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
        t_ensemble = -1

    # ensemble_name = "ensemble_" + rule.__doc__ + ".hdf"
    # part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble)

    max_cluster_size = myEAC.biggest_cluster_size(ensemble)

    logger.info("Maximum cluster size: {}".format(max_cluster_size))

    # # # # # # # # # # # # # #
    # check memory usage for different matrix schemes

    # compute memory usage for each type of matrix
    # linear properties for condensed sparse matrix
    n_s = 0.05
    n_e = 1.0
    val_s = 1.0
    val_e = 0.05

    ma = max_cluster_size * sparse_max_assocs_factor

    mems = compute_mems(n, ma, n_s, n_e, val_s, val_e)

    f_mat = mems[0] # full matrix
    fc_mat = mems[1] # full condensed matrix
    sp_const = mems[2] # sparse constant matrix
    sp_lin = mems[3] # sparse linear matrix

    sp_const_mst = mems[4]
    sp_lin_mst = mems[5]
예제 #2
0
            t.tic()
            ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
                                             n_partitions, n_iters)
            t.tac()
            part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
            logger.info("Saved ensemble in file: {}".format(ensemble_filename))
            t_ensemble = t.elapsed
        else:
            logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename))
            ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
            t_ensemble = -1

        # ensemble_name = "ensemble_" + rule.__doc__ + ".hdf"
        # part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble)

        max_cluster_size = myEAC.biggest_cluster_size(ensemble)

        logger.info("Maximum cluster size: {}".format(max_cluster_size))

        # # # # # # # # # # # # # #
        # check memory usage for different matrix schemes

        # compute memory usage for each type of matrix
        # linear properties for condensed sparse matrix
        n_s = 0.05
        n_e = 1.0
        val_s = 1.0
        val_e = 0.05

        ma = max_cluster_size * sparse_max_assocs_factor