def find_dataset(loader=None) -> pymongo.cursor.Cursor: d = {} if loader is not None: d['module'] = str(loader.__module__) + "." + str( loader.__class__.__name__) d.update(loader.get_dataset_info()) db = MongoDBConnection.get_datasets_db() return db.find(d)
def count_dataset(loader=None) -> int: d = {} if loader is not None: d['module'] = str(loader.__module__) + "." + str( loader.__class__.__name__) d.update(loader.get_dataset_info()) db = MongoDBConnection.get_datasets_db() return db.count_documents(d)
def save_dataset(dataset_name: str, loader): db = MongoDBConnection.get_datasets_db() dataset = db.find_one({"_id": dataset_name}) if dataset is None: info = dict(loader.get_dataset_info()) info['_id'] = dataset_name path = str(loader.__module__) + "." + str(loader.__class__.__name__) info['module'] = path db.insert_one(info) else: raise RuntimeError( "{0} already exists in the datasets database".format(dataset_name))
def get_dataset_obj(dataset_name: str): db = MongoDBConnection.get_datasets_db() dataset = db.find_one({"_id": dataset_name}) if dataset is not None: whole_path_str = dataset['module'] index = whole_path_str.rfind(".") module_str = whole_path_str[0:index] class_str = whole_path_str[index + 1:] module = importlib.import_module(module_str) my_class = getattr(module, class_str) return my_class.load_from_dict(dataset) else: raise RuntimeError("Unknown dataset {0}".format(dataset_name))
def remove_datasets(datasets: List[str]): db = MongoDBConnection.get_datasets_db() db.remove({"_id": {'$in': datasets}})
def update_datatse(dataset_name: str, loader): db = MongoDBConnection.get_datasets_db() db.update_one({"_id": dataset_name}, {"$set": loader.get_dataset_info()})
def get_dataset(dataset_name: str) -> Dict: db = MongoDBConnection.get_datasets_db() dataset = db.find_one({"_id": dataset_name}) return dataset