logger.info("New config: {}".format(rule.__doc__)) logger.info("* * * * * * * * * * * * * * * * * *") # skip if number of clusters is bigger than number of samples if n_clusts[1] >= n: logger.info("Kmax too large for dataset size. Skipping...") continue if n_clusts[0] <= 1: logger.info("Kmin too little. Skipping...") continue ## generate ensemble logger.info("Checking for ensemble in folder...") generator = myKM.K_Means(label_mode=lm, cuda_mem="manual") logger.info("No ensemble detected. Generating ensemble...") t.reset() t.tic() ensemble = part.generateEnsemble(data_sampled, generator, n_clusts, n_partitions, n_iters) t.tac() part.saveEnsembleToFileHDF(ensemble_path, ensemble) logger.info("Saved ensemble in file: {}".format(ensemble_path)) t_ensemble = t.elapsed del data_sampled, gt_sampled # end of dataset cycle
t = myProf.Timer() name = name + '_' ensemble_filename = os.path.join( folder, name + "ensemble_{}_{}.hdf".format(n, rule.__doc__)) if not os.path.exists(ensemble_filename): print "No ensemble detected. Generating ensemble..." generator = myKM.K_Means(cuda_mem="manual") n_clusts = rule(n) t.reset() t.tic() ensemble = part.generateEnsemble(data_sampled, generator, n_clusts, n_partitions, n_iters) t.tac() part.saveEnsembleToFileHDF(ensemble_filename, ensemble) print "Saved ensemble in file: {}".format(ensemble_filename) t_ensemble = t.elapsed else: print "Ensemble detected in file {}. Loading ensemble...".format( ensemble_filename) ensemble = part.loadEnsembleFromFileHDF(ensemble_filename) t_ensemble = -1 print "ensemble time:", t_ensemble max_cluster_size = part.biggest_cluster_size(ensemble) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # str_desc = "full condensed" print str_desc + " building matrix..."
rule = rule1 t = myProf.Timer() name = name + '_' ensemble_filename = os.path.join(folder,name + "ensemble_{}_{}.hdf".format(n, rule.__doc__)) if not os.path.exists(ensemble_filename): print "No ensemble detected. Generating ensemble..." generator = myKM.K_Means(cuda_mem="manual") n_clusts = rule(n) t.reset() t.tic() ensemble = part.generateEnsemble(data_sampled, generator, n_clusts, n_partitions, n_iters) t.tac() part.saveEnsembleToFileHDF(ensemble_filename, ensemble) print "Saved ensemble in file: {}".format(ensemble_filename) t_ensemble = t.elapsed else: print "Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename) ensemble = part.loadEnsembleFromFileHDF(ensemble_filename) t_ensemble = -1 print "ensemble time:", t_ensemble max_cluster_size = part.biggest_cluster_size(ensemble) # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # str_desc = "full condensed" print str_desc + " building matrix..." eacEst1 = myEAC.EAC(n_samples=n, sparse=False, condensed=True, n_partitions=n_partitions)
def get_ensemble(data_sampled, rule): n_clusts = rule(n) logger.info("* * * * * * * * * * * * * * * * * *") logger.info("Num. samples: {}".format(n)) logger.info("New config: {}".format(rule.__doc__)) logger.info("* * * * * * * * * * * * * * * * * *") # skip if number of clusters is bigger than number of samples if n_clusts[1] >= n: logger.info("Kmax too large for dataset size. Skipping...") continue if n_clusts[0] <= 1: logger.info("Kmin too little. Skipping...") continue ## generate ensemble logger.info("Checking for ensemble in folder...") generator = myKM.K_Means(cuda_mem="manual") # if there is an ensemble file load it, otherwise generate and save ensemble_filename = os.path.join(folder,"ensemble_{}_{}.hdf".format(n, rule.__doc__)) if not os.path.exists(ensemble_filename): logger.info("No ensemble detected. Generating ensemble...") t.reset() t.tic() ensemble = part.generateEnsemble(data_sampled, generator, n_clusts, n_partitions, n_iters) t.tac() part.saveEnsembleToFileHDF(ensemble_filename, ensemble) logger.info("Saved ensemble in file: {}".format(ensemble_filename)) t_ensemble = t.elapsed else: logger.info("Ensemble detected in file {}. Loading ensemble...".format(ensemble_filename)) ensemble = part.loadEnsembleFromFileHDF(ensemble_filename) t_ensemble = -1 # ensemble_name = "ensemble_" + rule.__doc__ + ".hdf" # part.saveEnsembleToFileHDF(os.path.join(folder, ensemble_name), ensemble) max_cluster_size = myEAC.biggest_cluster_size(ensemble) logger.info("Maximum cluster size: {}".format(max_cluster_size)) # # # # # # # # # # # # # # # check memory usage for different matrix schemes # compute memory usage for each type of matrix # linear properties for condensed sparse matrix n_s = 0.05 n_e = 1.0 val_s = 1.0 val_e = 0.05 ma = max_cluster_size * sparse_max_assocs_factor mems = compute_mems(n, ma, n_s, n_e, val_s, val_e) f_mat = mems[0] # full matrix fc_mat = mems[1] # full condensed matrix sp_const = mems[2] # sparse constant matrix sp_lin = mems[3] # sparse linear matrix sp_const_mst = mems[4] sp_lin_mst = mems[5]