generator = KMeans(init="random",n_init=1) # numpy generator
# if memRequired > 500:
# generator = KMeans(init="random",n_init=1) # numpy generator
# else:
# generator = K_Means() # cuda generator
data, gt = make_blobs(n_samples = n_samples, n_features = n_features, centers = centers)
data = data.astype(np.float32)
start = timer()
n_samples_sqrt = np.sqrt(n_samples)
n_clusters = [n_samples_sqrt / 2, n_samples_sqrt]
n_clusters = map(int,n_clusters)
ensemble = generateEnsemble(data, generator, n_clusters, iters = 3)
estimator = EAC(nsamples = n_samples,mat_sparse = False)
estimator.fit(ensemble, files = False)
elapsed = timer() - start
results[resultsIdx,0] = n_samples
results[resultsIdx,1] = n_features
sparsity = (estimator._coassoc.nonzero()[0].size - np.float(n_samples)) / (n_samples**2) # sparsity
results[resultsIdx,2] = sparsity
results[resultsIdx,3] = elapsed
resultsIdx += 1
print "round: ", r, "\ttook: ", elapsed, " seconds", "\tsparsity:", sparsity
logger.info("New config: {}".format(rule.__doc__))
logger.info("* * * * * * * * * * * * * * * * * *")
# skip if number of clusters is bigger than number of samples
if n_clusts[1] >= n:
logger.info("Kmax too large for dataset size. Skipping...")
continue
if n_clusts[0] <= 1:
logger.info("Kmin too little. Skipping...")
continue
## generate ensemble
logger.info("Checking for ensemble in folder...")
generator = myKM.K_Means(label_mode=lm, cuda_mem="manual")
logger.info("No ensemble detected. Generating ensemble...")
t.reset()
t.tic()
ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
n_partitions, n_iters)
t.tac()
part.saveEnsembleToFileHDF(ensemble_path, ensemble)
logger.info("Saved ensemble in file: {}".format(ensemble_path))
t_ensemble = t.elapsed
del data_sampled, gt_sampled
# end of dataset cycle
def get_ensemble(data_sampled, rule):
n_clusts = rule(n)
logger.info("* * * * * * * * * * * * * * * * * *")
logger.info("Num. samples: {}".format(n))
logger.info("New config: {}".format(rule.__doc__))
logger.info("* * * * * * * * * * * * * * * * * *")
# skip if number of clusters is bigger than number of samples
if n_clusts[1] >= n:
logger.info("Kmax too large for dataset size. Skipping...")
continue
if n_clusts[0] <= 1:
logger.info("Kmin too little. Skipping...")
continue
## generate ensemble
logger.info("Checking for ensemble in folder...")
generator = myKM.K_Means(cuda_mem="manual")
# if there is an ensemble file load it, otherwise generate and save
ensemble_filename = os.path.join(folder,"ensemble_{}_{}.hdf".format(n, rule.__doc__))
if not os.path.exists(ensemble_filename):
logger.info("No ensemble detected. Generating ensemble...")
t.reset()
t.tic()
ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
n_partitions, n_iters)
t.tac()
part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
logger.info("Saved ensemble in file: {}".format(ensemble_filename))
t_ensemble = t.elapsed
else:
logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename))
ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
t_ensemble = -1
# ensemble_name = "ensemble_" + rule.__doc__ + ".hdf"
# part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble)
max_cluster_size = myEAC.biggest_cluster_size(ensemble)
logger.info("Maximum cluster size: {}".format(max_cluster_size))
# # # # # # # # # # # # # #
# check memory usage for different matrix schemes
# compute memory usage for each type of matrix
# linear properties for condensed sparse matrix
n_s = 0.05
n_e = 1.0
val_s = 1.0
val_e = 0.05
ma = max_cluster_size * sparse_max_assocs_factor
mems = compute_mems(n, ma, n_s, n_e, val_s, val_e)
f_mat = mems[0] # full matrix
fc_mat = mems[1] # full condensed matrix
sp_const = mems[2] # sparse constant matrix
sp_lin = mems[3] # sparse linear matrix
sp_const_mst = mems[4]
sp_lin_mst = mems[5]