def get_population_snapshot(iterations_id: int, n_snapshot: int) -> np.array:
db_iter = MongoDBConnection.get_iterations_db()
iter_obj = db_iter.find_one({'_id': iterations_id})
populations_documents = MongoDBConnection.get_population_gridfs()
file_id = iter_obj['snapshots_population_documents'][n_snapshot]
with populations_documents.get(file_id) as f:
population_objs = np.load(f)
return population_objs
def get_benchmark_pareto(iteration_id: int) -> List[creator.Individual]:
db_iter = MongoDBConnection.get_benchmarks_iterations_db()
iter_obj = db_iter.find_one({'_id': iteration_id})
paretos_gridfs = MongoDBConnection.get_paretos_gridfs()
paretos = []
for pareto_file in iter_obj['paretos']:
with paretos_gridfs.get(pareto_file) as f:
paretos.append(pickle.loads(f.read()))
return paretos
def get_population(iterations_id: int) -> List[np.array]:
db_iter = MongoDBConnection.get_iterations_db()
iter_obj = db_iter.find_one({'_id': iterations_id})
populations_documents = MongoDBConnection.get_population_gridfs()
population_objs = []
for file_id in iter_obj['snapshots_population_documents']:
with populations_documents.get(file_id) as f:
population_objs.append(np.load(f))
return population_objs
def clear_population_performace_iteration(dataset_id=None):
find_d = {}
if dataset_id is not None:
find_d['dataset_id'] = dataset_id
db = MongoDBConnection.get_performance_populations_db()
iteration = db.find(find_d)
paretos_gridfs = MongoDBConnection.get_paretos_gridfs()
for it in iteration:
for pareto_file in it['paretos']:
paretos_gridfs.delete(pareto_file)
db.remove(find_d)
def save_population(iterations_id: int, whole_population: List[np.array]):
db = MongoDBConnection.get_population_gridfs()
gridfs_ids = [-1] * len(whole_population)
for n_snapshot, population in enumerate(whole_population):
with db.new_file() as f:
gridfs_ids[n_snapshot] = f._id
np.save(f, population)
db_iter = MongoDBConnection.get_iterations_db()
db_iter.update_one(
{'_id': iterations_id},
{'$set': {
'snapshots_population_documents': gridfs_ids
}})
def get_distintc_benchmark_iterations_id(dataset_id: str,
benchmark_id: str) -> List[str]:
db = MongoDBConnection.get_benchmarks_iterations_db()
return db.distinct('iteration_id', {
'dataset_id': dataset_id,
'benchmark_id': benchmark_id
})
def find_benchmark_iteration_by_id(iteration_id: str,
benchmark_id: str) -> pymongo.cursor.Cursor:
db = MongoDBConnection.get_benchmarks_iterations_db()
return db.find({
'iteration_id': iteration_id,
'benchmark_id': benchmark_id
})
def save_iteration(dataset_id: str, settings_id: str, cost: float,
solution: List[int], precision: List[float], n_calls: int,
duration: str, violations: int, historic: np.array) -> int:
db = MongoDBConnection.get_iterations_db()
result = db.insert_one({
"dataset_id":
dataset_id,
"settings_id":
settings_id,
"cost":
cost,
"solution":
solution,
"precision":
precision,
"n_calls":
n_calls,
"duration":
duration,
"violations":
violations,
"historic":
Binary(pickle.dumps(historic, protocol=2), subtype=128)
})
return result.inserted_id
def get_benchmark_performance_iteration(population_iteration: int,
bmk_settings: str) -> Dict:
db = MongoDBConnection.get_benchmark_performance_db()
iteration = db.find_one({
"population_iteration": population_iteration,
"bmk_settings": bmk_settings
})
return iteration
def clear_benchmark_iteration(dataset_id=None, benchmark_id=None):
find_d = {}
if dataset_id is not None:
find_d['dataset_id'] = dataset_id
if benchmark_id is not None:
find_d['benchmark_id'] = benchmark_id
db = MongoDBConnection.get_benchmarks_iterations_db()
iteration = db.find(find_d)
paretos_gridfs = MongoDBConnection.get_paretos_gridfs()
for it in iteration:
for pareto_file in it['paretos']:
paretos_gridfs.delete(pareto_file)
db.remove(find_d)
def find_dataset(loader=None) -> pymongo.cursor.Cursor:
d = {}
if loader is not None:
d['module'] = str(loader.__module__) + "." + str(
loader.__class__.__name__)
d.update(loader.get_dataset_info())
db = MongoDBConnection.get_datasets_db()
return db.find(d)
def count_dataset(loader=None) -> int:
d = {}
if loader is not None:
d['module'] = str(loader.__module__) + "." + str(
loader.__class__.__name__)
d.update(loader.get_dataset_info())
db = MongoDBConnection.get_datasets_db()
return db.count_documents(d)
def count_settings(ga_config=None) -> int:
d = {}
if ga_config is not None:
d['module'] = str(ga_config.__module__) + "." + str(
ga_config.__class__.__name__)
d.update(ga_config.make_dict())
db = MongoDBConnection.get_settings_db()
return db.count_documents(d)
def find_settings(ga_config=None) -> pymongo.cursor.Cursor:
d = {}
if ga_config is not None:
d['module'] = str(ga_config.__module__) + "." + str(
ga_config.__class__.__name__)
d.update(ga_config.make_dict())
db = MongoDBConnection.get_settings_db()
return db.find(d)
def save_benchmark_settings(settings_id: str, settings_dict: Dict):
db = MongoDBConnection.get_benchmark_settings_db()
settings = db.find_one({"_id": settings_id})
if settings is None:
settings_dict['_id'] = settings_id
db.insert_one(settings_dict)
else:
raise RuntimeError(
"{0} already exists in the settings database".format(settings_id))
def clear_benchmark_performace_iteration(population_iteration=None,
bmk_settings=None):
find_d = {}
if population_iteration is not None:
find_d['population_iteration'] = population_iteration
if bmk_settings is not None:
find_d['bmk_settings'] = bmk_settings
db = MongoDBConnection.get_benchmark_performance_db()
iteration = db.find(find_d)
paretos_gridfs = MongoDBConnection.get_paretos_gridfs()
for it in iteration:
for pareto_file in it['paretos']:
paretos_gridfs.delete(pareto_file)
db.remove(find_d)
def save_population_performance_iteration(
dataset_id: str, duration: List[str],
population: List[List[creator.Individual]]):
db = MongoDBConnection.get_paretos_gridfs()
paretos_pickles = [0] * len(population)
for n_snp, actual_pareto in enumerate(population):
with db.new_file() as f:
paretos_pickles[n_snp] = f._id
binary_data = Binary(pickle.dumps(actual_pareto, protocol=2),
subtype=128)
f.write(binary_data)
db = MongoDBConnection.get_performance_populations_db()
result = db.insert_one({
"dataset_id": dataset_id,
"duration": duration,
"paretos": paretos_pickles,
"version": "v1"
})
return result.inserted_id
def save_iteration(dataset_id: str, settings_id: str,
sp_generations: List[List[float]], paretos,
execution_info: Dict):
db = MongoDBConnection.get_paretos_gridfs()
paretos_pickles = [0] * len(paretos)
for n_snp, actual_pareto in enumerate(paretos):
with db.new_file() as f:
paretos_pickles[n_snp] = f._id
binary_data = Binary(pickle.dumps(actual_pareto, protocol=2),
subtype=128)
f.write(binary_data)
db = MongoDBConnection.get_iterations_db()
result = db.insert_one({
"dataset_id": dataset_id,
"settings_id": settings_id,
"execution_info": execution_info,
"snapshots": sp_generations,
"paretos": paretos_pickles,
"version": "v2"
})
return result.inserted_id
def save_benchmark_iteration(dataset_id: str, iteration_id: str,
benchmark_id: str, duration: List[str], paretos):
db = MongoDBConnection.get_paretos_gridfs()
paretos_pickles = [0] * len(paretos)
for n_snp, actual_pareto in enumerate(paretos):
with db.new_file() as f:
paretos_pickles[n_snp] = f._id
binary_data = Binary(pickle.dumps(actual_pareto, protocol=2),
subtype=128)
f.write(binary_data)
db = MongoDBConnection.get_benchmarks_iterations_db()
result = db.insert_one({
"dataset_id": dataset_id,
"iteration_id": iteration_id,
"benchmark_id": benchmark_id,
"duration": duration,
"paretos": paretos_pickles,
"version": "v1"
})
return result.inserted_id
def save_dataset(dataset_name: str, loader):
db = MongoDBConnection.get_datasets_db()
dataset = db.find_one({"_id": dataset_name})
if dataset is None:
info = dict(loader.get_dataset_info())
info['_id'] = dataset_name
path = str(loader.__module__) + "." + str(loader.__class__.__name__)
info['module'] = path
db.insert_one(info)
else:
raise RuntimeError(
"{0} already exists in the datasets database".format(dataset_name))
def get_settings_obj(settings_name: str):
db = MongoDBConnection.get_settings_db()
settings = db.find_one({"_id": settings_name})
if settings is not None:
whole_path_str = settings['module']
index = whole_path_str.rfind(".")
module_str = whole_path_str[0:index]
class_str = whole_path_str[index + 1:]
module = importlib.import_module(module_str)
my_class = getattr(module, class_str)
return my_class.load_from_dict(settings)
else:
raise RuntimeError("Unknown settings {0}".format(settings_name))
def get_dataset_obj(dataset_name: str):
db = MongoDBConnection.get_datasets_db()
dataset = db.find_one({"_id": dataset_name})
if dataset is not None:
whole_path_str = dataset['module']
index = whole_path_str.rfind(".")
module_str = whole_path_str[0:index]
class_str = whole_path_str[index + 1:]
module = importlib.import_module(module_str)
my_class = getattr(module, class_str)
return my_class.load_from_dict(dataset)
else:
raise RuntimeError("Unknown dataset {0}".format(dataset_name))
def save_settings(settings_name: str, ga_config):
db = MongoDBConnection.get_settings_db()
settings = db.find_one({"_id": settings_name})
if settings is None:
config_properties = dict(ga_config.serialize())
config_properties['_id'] = settings_name
path = str(ga_config.__module__) + "." + str(
ga_config.__class__.__name__)
config_properties['module'] = path
db.insert_one(config_properties)
else:
raise RuntimeError(
"{0} already exists in the settings database".format(
settings_name))
def get_settings(settings_name: str) -> Dict:
db = MongoDBConnection.get_settings_db()
settings = db.find_one({"_id": settings_name})
return settings
def remove_datasets(datasets: List[str]):
db = MongoDBConnection.get_datasets_db()
db.remove({"_id": {'$in': datasets}})
def update_datatse(dataset_name: str, loader):
db = MongoDBConnection.get_datasets_db()
db.update_one({"_id": dataset_name}, {"$set": loader.get_dataset_info()})
def get_available_pop_it_benchmark_performance(bmk_settings: str) -> List[int]:
db = MongoDBConnection.get_benchmark_performance_db()
pop_its = db.distinct("population_iteration",
{"bmk_settings": bmk_settings})
return pop_its
def count_population_performance_iterations(dataset_id: str) -> int:
db = MongoDBConnection.get_performance_populations_db()
n_iterations = db.count({"dataset_id": dataset_id})
return n_iterations
def get_population_performance_iterations(
dataset_id: str) -> pymongo.cursor.Cursor:
db = MongoDBConnection.get_performance_populations_db()
iterations = db.find({"dataset_id": dataset_id})
return iterations
def update_benchmark_settings(settings_id: str, settings_dict: Dict):
db = MongoDBConnection.get_benchmark_settings_db()
db.update_one({'_id': settings_id}, {'$set': settings_dict})