def find_dataset(loader=None) -> pymongo.cursor.Cursor:
d = {}
if loader is not None:
d['module'] = str(loader.__module__) + "." + str(
loader.__class__.__name__)
d.update(loader.get_dataset_info())
db = MongoDBConnection.get_datasets_db()
return db.find(d)
def count_dataset(loader=None) -> int:
d = {}
if loader is not None:
d['module'] = str(loader.__module__) + "." + str(
loader.__class__.__name__)
d.update(loader.get_dataset_info())
db = MongoDBConnection.get_datasets_db()
return db.count_documents(d)
def save_dataset(dataset_name: str, loader):
db = MongoDBConnection.get_datasets_db()
dataset = db.find_one({"_id": dataset_name})
if dataset is None:
info = dict(loader.get_dataset_info())
info['_id'] = dataset_name
path = str(loader.__module__) + "." + str(loader.__class__.__name__)
info['module'] = path
db.insert_one(info)
else:
raise RuntimeError(
"{0} already exists in the datasets database".format(dataset_name))
def get_dataset_obj(dataset_name: str):
db = MongoDBConnection.get_datasets_db()
dataset = db.find_one({"_id": dataset_name})
if dataset is not None:
whole_path_str = dataset['module']
index = whole_path_str.rfind(".")
module_str = whole_path_str[0:index]
class_str = whole_path_str[index + 1:]
module = importlib.import_module(module_str)
my_class = getattr(module, class_str)
return my_class.load_from_dict(dataset)
else:
raise RuntimeError("Unknown dataset {0}".format(dataset_name))
def remove_datasets(datasets: List[str]):
db = MongoDBConnection.get_datasets_db()
db.remove({"_id": {'$in': datasets}})
def update_datatse(dataset_name: str, loader):
db = MongoDBConnection.get_datasets_db()
db.update_one({"_id": dataset_name}, {"$set": loader.get_dataset_info()})
def get_dataset(dataset_name: str) -> Dict:
db = MongoDBConnection.get_datasets_db()
dataset = db.find_one({"_id": dataset_name})
return dataset