logger.info("New config: {}".format(rule.__doc__))
logger.info("* * * * * * * * * * * * * * * * * *")
# skip if number of clusters is bigger than number of samples
if n_clusts[1] >= n:
logger.info("Kmax too large for dataset size. Skipping...")
continue
if n_clusts[0] <= 1:
logger.info("Kmin too little. Skipping...")
continue
## generate ensemble
logger.info("Checking for ensemble in folder...")
generator = myKM.K_Means(label_mode=lm, cuda_mem="manual")
logger.info("No ensemble detected. Generating ensemble...")
t.reset()
t.tic()
ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
n_partitions, n_iters)
t.tac()
part.saveEnsembleToFileHDF(ensemble_path, ensemble)
logger.info("Saved ensemble in file: {}".format(ensemble_path))
t_ensemble = t.elapsed
del data_sampled, gt_sampled
# end of dataset cycle
t = myProf.Timer()
name = name + '_'
ensemble_filename = os.path.join(
folder, name + "ensemble_{}_{}.hdf".format(n, rule.__doc__))
if not os.path.exists(ensemble_filename):
print "No ensemble detected. Generating ensemble..."
generator = myKM.K_Means(cuda_mem="manual")
n_clusts = rule(n)
t.reset()
t.tic()
ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
n_partitions, n_iters)
t.tac()
part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
print "Saved ensemble in file: {}".format(ensemble_filename)
t_ensemble = t.elapsed
else:
print "Ensemble detected in file {}. Loading ensemble...".format(
ensemble_filename)
ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
t_ensemble = -1
print "ensemble time:", t_ensemble
max_cluster_size = part.biggest_cluster_size(ensemble)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
str_desc = "full condensed"
print str_desc + " building matrix..."
rule = rule1
t = myProf.Timer()
name = name + '_'
ensemble_filename = os.path.join(folder,name + "ensemble_{}_{}.hdf".format(n, rule.__doc__))
if not os.path.exists(ensemble_filename):
print "No ensemble detected. Generating ensemble..."
generator = myKM.K_Means(cuda_mem="manual")
n_clusts = rule(n)
t.reset()
t.tic()
ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
n_partitions, n_iters)
t.tac()
part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
print "Saved ensemble in file: {}".format(ensemble_filename)
t_ensemble = t.elapsed
else:
print "Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename)
ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
t_ensemble = -1
print "ensemble time:", t_ensemble
max_cluster_size = part.biggest_cluster_size(ensemble)
# # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
str_desc = "full condensed"
print str_desc + " building matrix..."
eacEst1 = myEAC.EAC(n_samples=n, sparse=False, condensed=True, n_partitions=n_partitions)
def get_ensemble(data_sampled, rule):
n_clusts = rule(n)
logger.info("* * * * * * * * * * * * * * * * * *")
logger.info("Num. samples: {}".format(n))
logger.info("New config: {}".format(rule.__doc__))
logger.info("* * * * * * * * * * * * * * * * * *")
# skip if number of clusters is bigger than number of samples
if n_clusts[1] >= n:
logger.info("Kmax too large for dataset size. Skipping...")
continue
if n_clusts[0] <= 1:
logger.info("Kmin too little. Skipping...")
continue
## generate ensemble
logger.info("Checking for ensemble in folder...")
generator = myKM.K_Means(cuda_mem="manual")
# if there is an ensemble file load it, otherwise generate and save
ensemble_filename = os.path.join(folder,"ensemble_{}_{}.hdf".format(n, rule.__doc__))
if not os.path.exists(ensemble_filename):
logger.info("No ensemble detected. Generating ensemble...")
t.reset()
t.tic()
ensemble = part.generateEnsemble(data_sampled, generator, n_clusts,
n_partitions, n_iters)
t.tac()
part.saveEnsembleToFileHDF(ensemble_filename, ensemble)
logger.info("Saved ensemble in file: {}".format(ensemble_filename))
t_ensemble = t.elapsed
else:
logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename))
ensemble = part.loadEnsembleFromFileHDF(ensemble_filename)
t_ensemble = -1
# ensemble_name = "ensemble_" + rule.__doc__ + ".hdf"
# part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble)
max_cluster_size = myEAC.biggest_cluster_size(ensemble)
logger.info("Maximum cluster size: {}".format(max_cluster_size))
# # # # # # # # # # # # # #
# check memory usage for different matrix schemes
# compute memory usage for each type of matrix
# linear properties for condensed sparse matrix
n_s = 0.05
n_e = 1.0
val_s = 1.0
val_e = 0.05
ma = max_cluster_size * sparse_max_assocs_factor
mems = compute_mems(n, ma, n_s, n_e, val_s, val_e)
f_mat = mems[0] # full matrix
fc_mat = mems[1] # full condensed matrix
sp_const = mems[2] # sparse constant matrix
sp_lin = mems[3] # sparse linear matrix
sp_const_mst = mems[4]
sp_lin_mst = mems[5]
