def get_population_snapshot(iterations_id: int, n_snapshot: int) -> np.array: db_iter = MongoDBConnection.get_iterations_db() iter_obj = db_iter.find_one({'_id': iterations_id}) populations_documents = MongoDBConnection.get_population_gridfs() file_id = iter_obj['snapshots_population_documents'][n_snapshot] with populations_documents.get(file_id) as f: population_objs = np.load(f) return population_objs
def get_benchmark_pareto(iteration_id: int) -> List[creator.Individual]: db_iter = MongoDBConnection.get_benchmarks_iterations_db() iter_obj = db_iter.find_one({'_id': iteration_id}) paretos_gridfs = MongoDBConnection.get_paretos_gridfs() paretos = [] for pareto_file in iter_obj['paretos']: with paretos_gridfs.get(pareto_file) as f: paretos.append(pickle.loads(f.read())) return paretos
def get_population(iterations_id: int) -> List[np.array]: db_iter = MongoDBConnection.get_iterations_db() iter_obj = db_iter.find_one({'_id': iterations_id}) populations_documents = MongoDBConnection.get_population_gridfs() population_objs = [] for file_id in iter_obj['snapshots_population_documents']: with populations_documents.get(file_id) as f: population_objs.append(np.load(f)) return population_objs
def clear_population_performace_iteration(dataset_id=None): find_d = {} if dataset_id is not None: find_d['dataset_id'] = dataset_id db = MongoDBConnection.get_performance_populations_db() iteration = db.find(find_d) paretos_gridfs = MongoDBConnection.get_paretos_gridfs() for it in iteration: for pareto_file in it['paretos']: paretos_gridfs.delete(pareto_file) db.remove(find_d)
def save_population(iterations_id: int, whole_population: List[np.array]): db = MongoDBConnection.get_population_gridfs() gridfs_ids = [-1] * len(whole_population) for n_snapshot, population in enumerate(whole_population): with db.new_file() as f: gridfs_ids[n_snapshot] = f._id np.save(f, population) db_iter = MongoDBConnection.get_iterations_db() db_iter.update_one( {'_id': iterations_id}, {'$set': { 'snapshots_population_documents': gridfs_ids }})
def get_distintc_benchmark_iterations_id(dataset_id: str, benchmark_id: str) -> List[str]: db = MongoDBConnection.get_benchmarks_iterations_db() return db.distinct('iteration_id', { 'dataset_id': dataset_id, 'benchmark_id': benchmark_id })
def find_benchmark_iteration_by_id(iteration_id: str, benchmark_id: str) -> pymongo.cursor.Cursor: db = MongoDBConnection.get_benchmarks_iterations_db() return db.find({ 'iteration_id': iteration_id, 'benchmark_id': benchmark_id })
def save_iteration(dataset_id: str, settings_id: str, cost: float, solution: List[int], precision: List[float], n_calls: int, duration: str, violations: int, historic: np.array) -> int: db = MongoDBConnection.get_iterations_db() result = db.insert_one({ "dataset_id": dataset_id, "settings_id": settings_id, "cost": cost, "solution": solution, "precision": precision, "n_calls": n_calls, "duration": duration, "violations": violations, "historic": Binary(pickle.dumps(historic, protocol=2), subtype=128) }) return result.inserted_id
def get_benchmark_performance_iteration(population_iteration: int, bmk_settings: str) -> Dict: db = MongoDBConnection.get_benchmark_performance_db() iteration = db.find_one({ "population_iteration": population_iteration, "bmk_settings": bmk_settings }) return iteration
def clear_benchmark_iteration(dataset_id=None, benchmark_id=None): find_d = {} if dataset_id is not None: find_d['dataset_id'] = dataset_id if benchmark_id is not None: find_d['benchmark_id'] = benchmark_id db = MongoDBConnection.get_benchmarks_iterations_db() iteration = db.find(find_d) paretos_gridfs = MongoDBConnection.get_paretos_gridfs() for it in iteration: for pareto_file in it['paretos']: paretos_gridfs.delete(pareto_file) db.remove(find_d)
def find_dataset(loader=None) -> pymongo.cursor.Cursor: d = {} if loader is not None: d['module'] = str(loader.__module__) + "." + str( loader.__class__.__name__) d.update(loader.get_dataset_info()) db = MongoDBConnection.get_datasets_db() return db.find(d)
def count_dataset(loader=None) -> int: d = {} if loader is not None: d['module'] = str(loader.__module__) + "." + str( loader.__class__.__name__) d.update(loader.get_dataset_info()) db = MongoDBConnection.get_datasets_db() return db.count_documents(d)
def count_settings(ga_config=None) -> int: d = {} if ga_config is not None: d['module'] = str(ga_config.__module__) + "." + str( ga_config.__class__.__name__) d.update(ga_config.make_dict()) db = MongoDBConnection.get_settings_db() return db.count_documents(d)
def find_settings(ga_config=None) -> pymongo.cursor.Cursor: d = {} if ga_config is not None: d['module'] = str(ga_config.__module__) + "." + str( ga_config.__class__.__name__) d.update(ga_config.make_dict()) db = MongoDBConnection.get_settings_db() return db.find(d)
def save_benchmark_settings(settings_id: str, settings_dict: Dict): db = MongoDBConnection.get_benchmark_settings_db() settings = db.find_one({"_id": settings_id}) if settings is None: settings_dict['_id'] = settings_id db.insert_one(settings_dict) else: raise RuntimeError( "{0} already exists in the settings database".format(settings_id))
def clear_benchmark_performace_iteration(population_iteration=None, bmk_settings=None): find_d = {} if population_iteration is not None: find_d['population_iteration'] = population_iteration if bmk_settings is not None: find_d['bmk_settings'] = bmk_settings db = MongoDBConnection.get_benchmark_performance_db() iteration = db.find(find_d) paretos_gridfs = MongoDBConnection.get_paretos_gridfs() for it in iteration: for pareto_file in it['paretos']: paretos_gridfs.delete(pareto_file) db.remove(find_d)
def save_population_performance_iteration( dataset_id: str, duration: List[str], population: List[List[creator.Individual]]): db = MongoDBConnection.get_paretos_gridfs() paretos_pickles = [0] * len(population) for n_snp, actual_pareto in enumerate(population): with db.new_file() as f: paretos_pickles[n_snp] = f._id binary_data = Binary(pickle.dumps(actual_pareto, protocol=2), subtype=128) f.write(binary_data) db = MongoDBConnection.get_performance_populations_db() result = db.insert_one({ "dataset_id": dataset_id, "duration": duration, "paretos": paretos_pickles, "version": "v1" }) return result.inserted_id
def save_iteration(dataset_id: str, settings_id: str, sp_generations: List[List[float]], paretos, execution_info: Dict): db = MongoDBConnection.get_paretos_gridfs() paretos_pickles = [0] * len(paretos) for n_snp, actual_pareto in enumerate(paretos): with db.new_file() as f: paretos_pickles[n_snp] = f._id binary_data = Binary(pickle.dumps(actual_pareto, protocol=2), subtype=128) f.write(binary_data) db = MongoDBConnection.get_iterations_db() result = db.insert_one({ "dataset_id": dataset_id, "settings_id": settings_id, "execution_info": execution_info, "snapshots": sp_generations, "paretos": paretos_pickles, "version": "v2" }) return result.inserted_id
def save_benchmark_iteration(dataset_id: str, iteration_id: str, benchmark_id: str, duration: List[str], paretos): db = MongoDBConnection.get_paretos_gridfs() paretos_pickles = [0] * len(paretos) for n_snp, actual_pareto in enumerate(paretos): with db.new_file() as f: paretos_pickles[n_snp] = f._id binary_data = Binary(pickle.dumps(actual_pareto, protocol=2), subtype=128) f.write(binary_data) db = MongoDBConnection.get_benchmarks_iterations_db() result = db.insert_one({ "dataset_id": dataset_id, "iteration_id": iteration_id, "benchmark_id": benchmark_id, "duration": duration, "paretos": paretos_pickles, "version": "v1" }) return result.inserted_id
def save_dataset(dataset_name: str, loader): db = MongoDBConnection.get_datasets_db() dataset = db.find_one({"_id": dataset_name}) if dataset is None: info = dict(loader.get_dataset_info()) info['_id'] = dataset_name path = str(loader.__module__) + "." + str(loader.__class__.__name__) info['module'] = path db.insert_one(info) else: raise RuntimeError( "{0} already exists in the datasets database".format(dataset_name))
def get_settings_obj(settings_name: str): db = MongoDBConnection.get_settings_db() settings = db.find_one({"_id": settings_name}) if settings is not None: whole_path_str = settings['module'] index = whole_path_str.rfind(".") module_str = whole_path_str[0:index] class_str = whole_path_str[index + 1:] module = importlib.import_module(module_str) my_class = getattr(module, class_str) return my_class.load_from_dict(settings) else: raise RuntimeError("Unknown settings {0}".format(settings_name))
def get_dataset_obj(dataset_name: str): db = MongoDBConnection.get_datasets_db() dataset = db.find_one({"_id": dataset_name}) if dataset is not None: whole_path_str = dataset['module'] index = whole_path_str.rfind(".") module_str = whole_path_str[0:index] class_str = whole_path_str[index + 1:] module = importlib.import_module(module_str) my_class = getattr(module, class_str) return my_class.load_from_dict(dataset) else: raise RuntimeError("Unknown dataset {0}".format(dataset_name))
def save_settings(settings_name: str, ga_config): db = MongoDBConnection.get_settings_db() settings = db.find_one({"_id": settings_name}) if settings is None: config_properties = dict(ga_config.serialize()) config_properties['_id'] = settings_name path = str(ga_config.__module__) + "." + str( ga_config.__class__.__name__) config_properties['module'] = path db.insert_one(config_properties) else: raise RuntimeError( "{0} already exists in the settings database".format( settings_name))
def get_settings(settings_name: str) -> Dict: db = MongoDBConnection.get_settings_db() settings = db.find_one({"_id": settings_name}) return settings
def remove_datasets(datasets: List[str]): db = MongoDBConnection.get_datasets_db() db.remove({"_id": {'$in': datasets}})
def update_datatse(dataset_name: str, loader): db = MongoDBConnection.get_datasets_db() db.update_one({"_id": dataset_name}, {"$set": loader.get_dataset_info()})
def get_available_pop_it_benchmark_performance(bmk_settings: str) -> List[int]: db = MongoDBConnection.get_benchmark_performance_db() pop_its = db.distinct("population_iteration", {"bmk_settings": bmk_settings}) return pop_its
def count_population_performance_iterations(dataset_id: str) -> int: db = MongoDBConnection.get_performance_populations_db() n_iterations = db.count({"dataset_id": dataset_id}) return n_iterations
def get_population_performance_iterations( dataset_id: str) -> pymongo.cursor.Cursor: db = MongoDBConnection.get_performance_populations_db() iterations = db.find({"dataset_id": dataset_id}) return iterations
def update_benchmark_settings(settings_id: str, settings_dict: Dict): db = MongoDBConnection.get_benchmark_settings_db() db.update_one({'_id': settings_id}, {'$set': settings_dict})