generator = KMeans(init="random",n_init=1) # numpy generator # if memRequired > 500: # generator = KMeans(init="random",n_init=1) # numpy generator # else: # generator = K_Means() # cuda generator data, gt = make_blobs(n_samples = n_samples, n_features = n_features, centers = centers) data = data.astype(np.float32) start = timer() n_samples_sqrt = np.sqrt(n_samples) n_clusters = [n_samples_sqrt / 2, n_samples_sqrt] n_clusters = map(int,n_clusters) ensemble = generateEnsemble(data, generator, n_clusters, iters = 3) estimator = EAC(nsamples = n_samples,mat_sparse = False) estimator.fit(ensemble, files = False) elapsed = timer() - start results[resultsIdx,0] = n_samples results[resultsIdx,1] = n_features sparsity = (estimator._coassoc.nonzero()[0].size - np.float(n_samples)) / (n_samples**2) # sparsity results[resultsIdx,2] = sparsity results[resultsIdx,3] = elapsed resultsIdx += 1 print "round: ", r, "\ttook: ", elapsed, " seconds", "\tsparsity:", sparsity
logger.info("New config: {}".format(rule.__doc__)) logger.info("* * * * * * * * * * * * * * * * * *") # skip if number of clusters is bigger than number of samples if n_clusts[1] >= n: logger.info("Kmax too large for dataset size. Skipping...") continue if n_clusts[0] <= 1: logger.info("Kmin too little. Skipping...") continue ## generate ensemble logger.info("Checking for ensemble in folder...") generator = myKM.K_Means(label_mode=lm, cuda_mem="manual") logger.info("No ensemble detected. Generating ensemble...") t.reset() t.tic() ensemble = part.generateEnsemble(data_sampled, generator, n_clusts, n_partitions, n_iters) t.tac() part.saveEnsembleToFileHDF(ensemble_path, ensemble) logger.info("Saved ensemble in file: {}".format(ensemble_path)) t_ensemble = t.elapsed del data_sampled, gt_sampled # end of dataset cycle
def get_ensemble(data_sampled, rule): n_clusts = rule(n) logger.info("* * * * * * * * * * * * * * * * * *") logger.info("Num. samples: {}".format(n)) logger.info("New config: {}".format(rule.__doc__)) logger.info("* * * * * * * * * * * * * * * * * *") # skip if number of clusters is bigger than number of samples if n_clusts[1] >= n: logger.info("Kmax too large for dataset size. Skipping...") continue if n_clusts[0] <= 1: logger.info("Kmin too little. Skipping...") continue ## generate ensemble logger.info("Checking for ensemble in folder...") generator = myKM.K_Means(cuda_mem="manual") # if there is an ensemble file load it, otherwise generate and save ensemble_filename = os.path.join(folder,"ensemble_{}_{}.hdf".format(n, rule.__doc__)) if not os.path.exists(ensemble_filename): logger.info("No ensemble detected. Generating ensemble...") t.reset() t.tic() ensemble = part.generateEnsemble(data_sampled, generator, n_clusts, n_partitions, n_iters) t.tac() part.saveEnsembleToFileHDF(ensemble_filename, ensemble) logger.info("Saved ensemble in file: {}".format(ensemble_filename)) t_ensemble = t.elapsed else: logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename)) ensemble = part.loadEnsembleFromFileHDF(ensemble_filename) t_ensemble = -1 # ensemble_name = "ensemble_" + rule.__doc__ + ".hdf" # part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble) max_cluster_size = myEAC.biggest_cluster_size(ensemble) logger.info("Maximum cluster size: {}".format(max_cluster_size)) # # # # # # # # # # # # # # # check memory usage for different matrix schemes # compute memory usage for each type of matrix # linear properties for condensed sparse matrix n_s = 0.05 n_e = 1.0 val_s = 1.0 val_e = 0.05 ma = max_cluster_size * sparse_max_assocs_factor mems = compute_mems(n, ma, n_s, n_e, val_s, val_e) f_mat = mems[0] # full matrix fc_mat = mems[1] # full condensed matrix sp_const = mems[2] # sparse constant matrix sp_lin = mems[3] # sparse linear matrix sp_const_mst = mems[4] sp_lin_mst = mems[5]