def put(self, schema: str, data: dict) -> list:
"""
Insert data to mongodb
:param schema: schema (name of collection in mongodb)
:type schema: str
:param data: data to store in dictionary format
:type data: dict
:return: list of dictionary with inserted _id
:rtype: list
"""
# Create the output list
output_list = list()
try:
# Get collection creating it if not exists
mongo_collect = self._get_collection(schema,
create_collection=True)
# Insert data and recover _id
mongo_collect.insert_one(data)
output_list.append(
{AccessDatabase.ID_FIELD: data[AccessDatabase.ID_FIELD]})
# Return the list with the updated elements
return output_list
except ServerSelectionTimeoutError:
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
except Exception:
raise DatabaseObjectException(ErrorMessages.PUT_ERROR)
def update(self, schema: str, data: dict, conditions: list, criteria: str,
native_criteria: bool) -> list:
"""
Update data in mongodb
:param schema: schema (name of collection in mongodb)
:type schema: str
:param data: data to store in dictionary format
:type data: dict
:param conditions: conditions to search (list of tuple)
:type conditions: list
:param criteria: criteria from mongodb
:type criteria: str
:param native_criteria: boolean for search by native criteria from mongodb
:type native_criteria: bool
:return: list of dictionary with number of updated elements
:rtype: list
"""
# Create the output list
output_list = list()
try:
# Get collection from mongodb
mongo_collect = self._get_collection(schema)
# Define data to update
mongo_data_update = {
AccessDatabaseMongoDB.MONGO_UPDATE_OPERATOR: data
}
# Get criteria in mongodb language
mongo_criteria = AccessDatabaseMongoDB._create_mongo_criteria(
conditions, criteria, native_criteria)
# Update elements and recover number of updated elements and number of matched elements
mongo_result = mongo_collect.update_many(mongo_criteria,
mongo_data_update)
modified_count = mongo_result.modified_count
# Add number of updated elements
output_list.append({AccessDatabase.UPDATED_COUNT: modified_count})
# Return the list with the updated elements
return output_list
except ServerSelectionTimeoutError:
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
except Exception:
raise DatabaseObjectException(ErrorMessages.PUT_ERROR)
def remove(self, schema: str, conditions: list, criteria: str,
native_criteria: bool) -> list:
"""
Delete elements from mongodb
:param schema: schema (name of collection in mongodb)
:type schema: str
:param conditions: conditions to search (list of tuple)
:type conditions: list
:param criteria: criteria from mongodb
:type criteria: str
:param native_criteria: boolean for search by native criteria from mongodb
:type native_criteria: bool
:return: list of dictionary with number of deleted elements
:rtype: list
"""
# Create the output list
output_list = list()
try:
# Get collection from mongodb
mongo_collect = self._get_collection(schema)
# Get criteria in mongodb language
mongo_criteria = AccessDatabaseMongoDB._create_mongo_criteria(
conditions, criteria, native_criteria)
# Delete elements with criteria and recover number of deleted elements
mongo_result = mongo_collect.delete_many(mongo_criteria)
deleted_count = mongo_result.deleted_count
# Add number of deleted elements
output_list.append({AccessDatabase.DELETED_COUNT: deleted_count})
# Return the list with removed elements.
return output_list
except ServerSelectionTimeoutError:
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
except Exception as e:
# If there is SCHEMA_ERROR, this is not an error because schema does not exist. Only log warning
if str(e) == ErrorMessages.SCHEMA_ERROR:
logger.warning('Schema or subschema does not exist')
output_list.append({AccessDatabase.DELETED_COUNT: 0})
return output_list
else:
raise DatabaseObjectException(ErrorMessages.REMOVE_ERROR)
def get(self, schema: str, conditions: list, criteria: str,
native_criteria: bool) -> list:
"""
Get data from mongodb
:param schema: schema (name of collection in mongodb)
:type schema: str
:param conditions: conditions to search (list of tuple)
:type conditions: list
:param criteria: criteria from mongodb
:type criteria: str
:param native_criteria: boolean for search by native criteria from mongodb
:type native_criteria: bool
:return: list of dictionary with data
:rtype: list
"""
# Create the output list
output_list = list()
try:
# Get collection
mongo_collect = self._get_collection(schema)
# Get criteria in mongodb language
mongo_criteria = AccessDatabaseMongoDB._create_mongo_criteria(
conditions, criteria, native_criteria)
# Find data with criteria
mongo_result = mongo_collect.find(mongo_criteria).sort(
AccessDatabase.TIMESTAMP_FIELD)
# For each data: recover _id, set to data and add to output list
for element in mongo_result:
del element[AccessDatabaseMongoDB.OBJECT_ID_FIELD]
output_list.append(element)
# Return the list with the updated elements
return output_list
except ServerSelectionTimeoutError:
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
except Exception:
raise DatabaseObjectException(ErrorMessages.GET_ERROR)
def put_object(self, schema: str, sub_schema: str,
data: DatabaseObject) -> DatabaseObjectResult:
"""
Write object to object store
:param schema: connection schema
:type schema: str
:param sub_schema: object type to save
:type sub_schema: str
:param data: data to save
:type data: DatabaseObject
:return: database object result with _id
:rtype: DatabaseObjectResult
"""
# Validate data, checking that has inheritance from DatabaseObject
if not issubclass(data.__class__, DatabaseObject):
e = DatabaseObjectException(ErrorMessages.INHERITANCE_ERROR)
return DatabaseObjectModule._get_data_object_result_from_json(
'put', exception=e)
return self.put(schema, sub_schema, data.__dict__)
def __init__(self, connection_url: str) -> None:
"""
Constructor with url connection
:param connection_url: url connection from ini file
:type connection_url: str
:return: This function return nothing
:rtype: None
"""
# Init the father class
AccessDatabase.__init__(self, connection_url)
try:
# Connect with mongodb
self.open_connection()
# Get database from URL connection
self.db = self.connection.get_database()
# Initialize caches
self.cache_collections = dict()
except errors.ConfigurationError:
raise DatabaseObjectException(ErrorMessages.CONFIGURATION_ERROR)
except DatabaseObjectException as e:
raise e
def put(self, schema: str, sub_schema: str,
data: dict) -> DatabaseObjectResult:
"""
Write object to object store
:param schema: connection schema
:type schema: str
:param sub_schema: object type to save
:type sub_schema: str
:param data: data to save
:type data: dict
:return: database object result with _id
:rtype: DatabaseObjectResult
"""
try:
# Check if database is up
if not self.is_connected:
logger.error('Error opening connection to datastore',
exc_info=True)
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
# Validate data, checking that object has mandatory fields
self._validate_data(data)
# Recover collection joining schema and sub_schema, same for index
schema_collection = AccessDatabase.get_schema_collection(
schema, sub_schema)
schema_collection_index = AccessDatabase.get_schema_collection_index(
schema, sub_schema)
# Get next index and set index to data
next_index = self._get_next_index(data, schema, sub_schema)
data[AccessDatabase.ID_FIELD] = next_index
# Set timestamp attribute
data[AccessDatabase.TIMESTAMP_FIELD] = int(time.time() * 10000000)
ret = self.access_db.put(schema_collection, data)
# Update cache index
self.cache_index[schema_collection_index] = next_index
self.access_db.update_index(schema_collection_index, next_index)
return DatabaseObjectModule._get_data_object_result_from_json(
'put', result=ret)
except DatabaseObjectException as e:
# Set false only if socket timeout exception and if another process has not set it to false
if str(e) == ErrorMessages.CONNECTION_ERROR and self.is_connected:
self.is_connected = False
logger.critical('Connection to datastore lost')
logger.error('Error inserting data to datastore', exc_info=True)
return DatabaseObjectModule._get_data_object_result_from_json(
'put', exception=e)
def _get_next_index(self, data: dict, schema: str, sub_schema: str) -> int:
"""
Check index of data to insert
:param data: data to insert
:param schema: schema to search
:param sub_schema: sub_schema to search
:return: next index of the data or exception
"""
# Recover collection index joining schema and sub_schema
schema_collection_index = AccessDatabase.get_schema_collection_index(
schema, sub_schema)
# Check that schema_collections exists, and if not then create in cache_index
if schema_collection_index not in self.cache_index.keys():
self.cache_index[
schema_collection_index] = self.access_db.get_last_index(
schema, sub_schema)
# If not exists _identifier, then create a new _identifier from cache
# If exists _identifier, then check that is greather than last inserted _identifier. If not then raise excepcion
if data[AccessDatabase.ID_FIELD] is None:
next_index = self.cache_index[schema_collection_index] + 1
else:
next_index = data[AccessDatabase.ID_FIELD]
if next_index <= self.cache_index[schema_collection_index]:
raise DatabaseObjectException(ErrorMessages.INDEX_VALUE_ERROR)
return next_index
def get_last_index(self, schema: str, sub_schema: str) -> int:
"""
Recover last index from all collections
:param schema: schema to search
:param sub_schema: sub_schema to search
:return: last inserted index of schema_collection
"""
schema_collection = self.get_schema_collection(schema, sub_schema)
schema_collection_index = self.get_schema_collection_index(
schema, sub_schema)
try:
# Recover collections. Add collection to cache if not exists
collection_index = self._get_collection(schema_collection_index,
create_collection=True,
create_index=False)
collection_data = self._get_collection(schema_collection,
create_collection=True)
# Find last data inserted from data collection
sort_filter = '{' + AccessDatabaseMongoDB.OBJECT_ID_FIELD + ':-1}'
mongo_result = collection_data.find({}).sort(sort_filter).limit(1)
result = [
element[AccessDatabase.ID_FIELD] for element in mongo_result
]
identifier_collection = 0 if len(result) == 0 else result[0]
# Find last index inserted from index collection
mongo_result = collection_index.find({})
result = [
element[AccessDatabase.ID_FIELD] for element in mongo_result
]
identifier_collection_index = 0 if len(result) == 0 else result[0]
return max(identifier_collection, identifier_collection_index)
except ServerSelectionTimeoutError:
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
except Exception:
raise DatabaseObjectException(ErrorMessages.GET_INDEX_ERROR)
def update_index(self, schema_collection_index: str, value: int) -> None:
try:
# Get collection from mongodb
mongo_collect = self._get_collection(schema_collection_index,
create_collection=True,
create_index=False)
# Define data to update
mongo_data = {AccessDatabase.ID_FIELD: value}
mongo_data_update = {
AccessDatabaseMongoDB.MONGO_UPDATE_OPERATOR: mongo_data
}
# Update all elements of collection index
mongo_collect.update_many({}, mongo_data_update, upsert=True)
except ServerSelectionTimeoutError:
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
except Exception:
raise DatabaseObjectException(ErrorMessages.UPDATE_INDEX_ERROR)
def get(self,
schema: str,
sub_schema: str,
conditions: list = ((AccessDatabase.ID_FIELD, '!=', None), ),
criteria: str = '',
native_criteria: bool = False) -> DatabaseObjectResult:
"""
Get data from data store
:param schema: connection schema
:type schema: str
:param sub_schema: object type to save
:type sub_schema: str
:param conditions: list of tuple conditions
:type conditions: list
:param criteria: criteria search
:type criteria: str
:param native_criteria: criteria native from database
:type native_criteria: bool
:return: database object result
:rtype: DatabaseObjectResult
"""
try:
# Check if database is up
if not self.is_connected:
logger.error('Error opening connection to datastore',
exc_info=True)
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
schema_collection = AccessDatabase.get_schema_collection(
schema, sub_schema)
ret = self.access_db.get(schema_collection, conditions, criteria,
native_criteria)
return DatabaseObjectModule._get_data_object_result_from_json(
'get', result=ret)
except DatabaseObjectException as e:
# Set false only if socket timeout exception and if another process has not set it to false
if str(e) == ErrorMessages.CONNECTION_ERROR and self.is_connected:
self.is_connected = False
logger.critical('Connection to datastore lost')
logger.error('Error recovering data from datastore', exc_info=True)
return DatabaseObjectModule._get_data_object_result_from_json(
'get', exception=e)
def _validate_data(data: dict) -> None:
"""
Check of data inherit from DatabaseObject
:param data: data to check
:type data: dict
:return: This function return nothing
:rtype: None
"""
if not all(key in data.keys() for key in vars(DatabaseObject())):
raise DatabaseObjectException(ErrorMessages.INHERITANCE_ERROR)
def open_connection(self) -> None:
"""
Get the connection whith mongodb and check that it is successful.
:return: object that represents the connection
:rtype: object
"""
try:
self.connection = MongoClient(self.connection_url)
self.connection.is_mongos
except (errors.ConnectionFailure, errors.ServerSelectionTimeoutError):
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
def _get_collection(self, schema: str, create_collection: bool = False, create_index: bool = True) \
-> collection.Collection:
"""
Get collection from database
:param schema: collection of mongodb
:type schema: str
:param create_collection: create collection or not
:type create_collection: bool
:return: collection from mongodb
:rtype: collection
"""
# If collection exists, get it.
if schema in self.cache_collections.keys():
mongo_collect = self.cache_collections[schema]
# If not, if collection is in mongodb, get and cache
elif schema in self.db.collection_names():
mongo_collect = self.db[schema]
self.cache_collections[schema] = mongo_collect
# If not, if the function can create it, create it
elif create_collection:
mongo_collect = self.db.create_collection(schema)
if create_index:
mongo_collect.create_index(
[(AccessDatabase.TIMESTAMP_FIELD, ASCENDING)],
name=AccessDatabase.TIMESTAMP_FIELD,
unique=True)
mongo_collect.create_index(
[(AccessDatabase.ID_FIELD, ASCENDING)],
name=AccessDatabase.ID_FIELD,
unique=True)
self.cache_collections[schema] = mongo_collect
# In other case, throw the exception
else:
logger.warning(
'Mongodb schema {} can not be recovered'.format(schema))
raise DatabaseObjectException(ErrorMessages.SCHEMA_ERROR)
# Return the mongo collection
return mongo_collect
def _implement_database(name_database: str, connection_database: str) -> AccessDatabase:
"""
Set implemented database
:param name_database: name of implemented database
:type name_database: str
:param connection_database: url connection
:type connection_database: str
:return: instance of implemented database
:rtype: AccessDatabase
"""
if name_database == 'mongodb':
return AccessDatabaseMongoDB(connection_database)
else:
raise DatabaseObjectException(ErrorMessages.CONFIGURATION_ERROR)
def update_object(self,
schema: str,
sub_schema: str,
data: DatabaseObject,
conditions: list = ((AccessDatabase.ID_FIELD, '!=',
None), ),
criteria: str = '',
native_criteria: bool = False) -> DatabaseObjectResult:
"""
Update data in object store. This method will update all object data defined in "data" attribute
:param schema: connection schema
:type schema: str
:param sub_schema: object type to save
:type sub_schema: str
:param data: data to update
:type data: DatabaseObject
:param conditions: list of tuple conditions
:type conditions: list
:param criteria: criteria to search
:type criteria: str
:param native_criteria: use native criteria from database or not
:type native_criteria: bool
:return: data object result wit number of updated objects
:rtype: DatabaseObjectResult
"""
# Validate data, checking that has inheritance from DatabaseObject
if not issubclass(data.__class__, DatabaseObject):
e = DatabaseObjectException(ErrorMessages.INHERITANCE_ERROR)
return DatabaseObjectModule._get_data_object_result_from_json(
'update', exception=e)
# This updates all object data defined in "data" attribute because we pass __dict__ to update method
return self.update(schema, sub_schema, data.__dict__, conditions,
criteria, native_criteria)
def _create_mongo_criteria(conditions: list, criteria: str,
native_criteria: bool) -> dict:
"""
Create criteria native of mongodb
:param conditions: list of tuple of conditions
:type conditions: list
:param criteria: native criteria language from mongodb
:type criteria: str
:param native_criteria: bool for use native criteria or not
:type native_criteria: bool
:return: filter
:rtype: dict
"""
# Create a filter with list of empty conditions
mongo_criteria = {AccessDatabaseMongoDB.MONGO_JOIN_CONDITION: list()}
try:
# Iterate all conditions to create native mongodb filter
for condition in conditions:
# Special case if the variable is the ID
if condition[0] == AccessDatabase.ID_FIELD:
# If the value to compare is a list, one list of ObjectId must be generated
if isinstance(condition[2], (list, tuple)):
filter_condition = {
condition[0]: {
AccessDatabaseMongoDB.MONGO_OPERATORS[condition[1]]:
conditions[2]
}
}
# If the value is zero, negative or None, all elements must be recovered
elif condition[2] is None or condition[2] <= 0:
filter_condition = {}
# Else the ObjectId must be generated
else:
filter_condition = {
condition[0]: {
AccessDatabaseMongoDB.MONGO_OPERATORS[condition[1]]:
condition[2]
}
}
# The rest of the variables do not have special cases
else:
value_compare = condition[2]
filter_condition = {
condition[0]: {
AccessDatabaseMongoDB.MONGO_OPERATORS[condition[1]]:
value_compare
}
}
# Add condition translated to mongodb filter language
mongo_criteria[AccessDatabaseMongoDB.
MONGO_JOIN_CONDITION].append(filter_condition)
# Only if native criteria is active, add native from user
if native_criteria and len(criteria) > 0:
mongo_criteria[
AccessDatabaseMongoDB.MONGO_JOIN_CONDITION].append(
dict(criteria))
# Return the mongo criteria
logger.debug('Mongo criteria {}'.format(mongo_criteria))
return mongo_criteria
except Exception:
raise DatabaseObjectException(ErrorMessages.CRITERIA_ERROR)
def update(self,
schema: str,
sub_schema: str,
data: dict,
conditions: list = ((AccessDatabase.ID_FIELD, '!=', ''), ),
criteria: str = '',
native_criteria: bool = False) -> DatabaseObjectResult:
"""
Update data in object store. . This method will update only fields defined in "data" attribute
:param schema: connection schema
:type schema: str
:param sub_schema: object type to save
:type sub_schema: str
:param data: data to update
:type data: dict
:param conditions: list of tuple conditions
:type conditions: list
:param criteria: criteria to search
:type criteria: str
:param native_criteria: use native criteria from database or not
:type native_criteria: bool
:return: data object result wit number of updated objects
:rtype: DatabaseObjectResult
"""
try:
# Check if database is up
if not self.is_connected:
logger.error('Error opening connection to datastore',
exc_info=True)
raise DatabaseObjectException(ErrorMessages.CONNECTION_ERROR)
# Delete all private attributes. It is forbidden to modify this attributes
del data[AccessDatabase.ID_FIELD]
del data[AccessDatabase.TIMESTAMP_FIELD]
del data[AccessDatabase.UPDATED_COUNT]
del data[AccessDatabase.DELETED_COUNT]
schema_collection = AccessDatabase.get_schema_collection(
schema, sub_schema)
ret = self.access_db.update(schema_collection, data, conditions,
criteria, native_criteria)
return DatabaseObjectModule._get_data_object_result_from_json(
'update', result=ret)
except DatabaseObjectException as e:
# Set false only if socket timeout exception and if another process has not set it to false
if str(e) == ErrorMessages.CONNECTION_ERROR and self.is_connected:
self.is_connected = False
logger.critical('Connection to datastore lost')
logger.error('Error updating data from datastore', exc_info=True)
return DatabaseObjectModule._get_data_object_result_from_json(
'update', exception=e)