generator = KMeans(init="random",n_init=1) # numpy generator
            # if memRequired > 500:
            #     generator = KMeans(init="random",n_init=1) # numpy generator
            # else:
            #     generator = K_Means() # cuda generator

            data, gt = make_blobs(n_samples = n_samples, n_features = n_features, centers = centers)
            data = data.astype(np.float32)

            start = timer()

            n_samples_sqrt = np.sqrt(n_samples)
            n_clusters = [n_samples_sqrt / 2, n_samples_sqrt]
            n_clusters = map(int,n_clusters)

            ensemble = generateEnsemble(data, generator, n_clusters, iters = 3)
            
            estimator = EAC(nsamples = n_samples,mat_sparse = False)
            estimator.fit(ensemble, files = False)

            elapsed = timer() - start

            results[resultsIdx,0] = n_samples
            results[resultsIdx,1] = n_features
            sparsity = (estimator._coassoc.nonzero()[0].size - np.float(n_samples)) / (n_samples**2) # sparsity
            results[resultsIdx,2] = sparsity
            results[resultsIdx,3] = elapsed
            resultsIdx += 1

            print "round: ", r, "\ttook: ", elapsed, " seconds", "\tsparsity:", sparsity
            
Example #2
0
        logger.info("New config: {}".format(rule.__doc__))
        logger.info("* * * * * * * * * * * * * * * * * *")

        # skip if number of clusters is bigger than number of samples
        if n_clusts[1] >= n:
            logger.info("Kmax too large for dataset size. Skipping...")
            continue
        if n_clusts[0] <= 1:
            logger.info("Kmin too little. Skipping...")
            continue            


        ## generate ensemble
        logger.info("Checking for ensemble in folder...")


        generator = myKM.K_Means(label_mode=lm, cuda_mem="manual")


        logger.info("No ensemble detected. Generating ensemble...")
        t.reset()
        t.tic()
        ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
                                         n_partitions, n_iters)
        t.tac()
        part.saveEnsembleToFileHDF(ensemble_path, ensemble)
        logger.info("Saved ensemble in file: {}".format(ensemble_path))
        t_ensemble = t.elapsed

    del data_sampled, gt_sampled
    # end of dataset cycle
Example #3
0
def get_ensemble(data_sampled, rule):
    n_clusts = rule(n)

    logger.info("* * * * * * * * * * * * * * * * * *")
    logger.info("Num. samples: {}".format(n))
    logger.info("New config: {}".format(rule.__doc__))
    logger.info("* * * * * * * * * * * * * * * * * *")

    # skip if number of clusters is bigger than number of samples
    if n_clusts[1] >= n:
        logger.info("Kmax too large for dataset size. Skipping...")
        continue
    if n_clusts[0] <= 1:
        logger.info("Kmin too little. Skipping...")
        continue            


    ## generate ensemble
    logger.info("Checking for ensemble in folder...")

    generator = myKM.K_Means(cuda_mem="manual")    

    # if there is an ensemble file load it, otherwise generate and save
    ensemble_filename = os.path.join(folder,"ensemble_{}_{}.hdf".format(n, rule.__doc__))
    if not os.path.exists(ensemble_filename):
        logger.info("No ensemble detected. Generating ensemble...")
        t.reset()
        t.tic()
        ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
                                         n_partitions, n_iters)
        t.tac()
        part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
        logger.info("Saved ensemble in file: {}".format(ensemble_filename))
        t_ensemble = t.elapsed
    else:
        logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename))
        ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
        t_ensemble = -1

    # ensemble_name = "ensemble_" + rule.__doc__ + ".hdf"
    # part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble)

    max_cluster_size = myEAC.biggest_cluster_size(ensemble)

    logger.info("Maximum cluster size: {}".format(max_cluster_size))

    # # # # # # # # # # # # # #
    # check memory usage for different matrix schemes

    # compute memory usage for each type of matrix
    # linear properties for condensed sparse matrix
    n_s = 0.05
    n_e = 1.0
    val_s = 1.0
    val_e = 0.05

    ma = max_cluster_size * sparse_max_assocs_factor

    mems = compute_mems(n, ma, n_s, n_e, val_s, val_e)

    f_mat = mems[0] # full matrix
    fc_mat = mems[1] # full condensed matrix
    sp_const = mems[2] # sparse constant matrix
    sp_lin = mems[3] # sparse linear matrix

    sp_const_mst = mems[4]
    sp_lin_mst = mems[5]