def execute(self):
# Set the connection to the database
self.get_sqlmanager_instance().set_db_settings(self.db_settings)
# Check the integrity of the database
str_ok = self.get_sqlmanager_instance().check_database_str_integrity()
if (not str_ok):
raise DenCellORFException(
'The schema of the database provided does not follow' +
' the expected model. Please make sure the provided model (' +
self.db_model + ') and the database (' +
self.db_settings[Constants.DB_SETTINGS_DB_NAME] +
') provided are the right ones.')
# Get the declarative base corresponding to the database
base = self.get_sqlmanager_instance().get_declarative_base()
# Build a dictionary of the classes defined in the model
# where the keys are the classes, and the values their names
dict_model_classes = {}
for (cl_name, cl_object) in base._decl_class_registry.items():
if (not str(cl_name) == '_sa_module_registry'):
dict_model_classes[cl_object] = str(cl_name)
# For each table, get the list of all entries and save them in a file
for table in dict_model_classes.keys():
# Get the name of the tale
table_name = str(dict_model_classes[table])
Logger.get_instance().debug(
'Starting to save the entries of the ' + table_name +
' table.')
# Get all the entries to save
objects_to_save = self.get_sqlmanager_instance().get_session(
).query(table).all()
# Expunge the session to the database to detach the objects in the list from the session
self.get_sqlmanager_instance().get_session().expunge_all()
self.get_sqlmanager_instance().close_session()
if self.file_prefix:
filename = self.file_prefix + table_name
else:
filename = table_name
try:
FileHandlerUtil.save_obj_to_file(
objects_to_save=objects_to_save,
filename=filename,
output_folder=self.output_folder)
except Exception as e:
raise DenCellORFException(
'BackupStrategy.execute(): An error occurred trying to' +
' save data in the file.'
'\n Error code: ' + LogCodes.ERR_FILEHAND + '.', e)
def __init__( self, message, exception=None, dna_seq=None, aa_seq=None, remainder=None ):
DenCellORFException.__init__( self,
message = message,
exception = exception )
self.dna_seq = dna_seq
self.aa_seq = aa_seq
self.remainder = remainder
def rollback_session(self):
try:
self.get_session().rollback()
except exc.SQLAlchemyError as e:
raise DenCellORFException(
self.classname +
'.rollback(): A SQLAlchemy error occurred trying to' +
' rollback the session.', e)
except Exception as e:
raise DenCellORFException(
self.classname + '.rollback(): An error occurred trying to' +
' rollback the session.', e)
def __init__(self, message, exception=None, code=None):
DenCellORFException.__init__(self,
message=message,
exception=exception)
if code:
self.code = code
else:
try:
self.code = exception.code
except:
self.code = None
def save_obj_to_file( objects_to_save, filename, output_folder=DefaultOutputFolder.OUTPUT_FOLDER ):
# Create the output folder if it does not yet exist
# (and its parent folders if necessary)
if not os.path.isdir( output_folder ):
os.makedirs( output_folder )
file_path = os.path.join( output_folder, filename ) + Constants.DENCELLORF_FILES_EXTENSION
Logger.get_instance().debug( 'FileHandlerUtil.save_obj_to_file(): ' +
str( len( objects_to_save ) ) +
' objects will be saved in ' + file_path + '.' )
# Save the objects in the file
try:
with open( file_path, 'wb' ) as objects_to_save_file:
obj_to_insert_pickler = pickle.Pickler( objects_to_save_file )
obj_to_insert_pickler.dump( objects_to_save )
except Exception as e:
raise DenCellORFException( 'FileHandlerUtil.save_obj_to_file(): An error occurred trying' +
' to save the objects in ' + file_path +
'. Hence, these data will not be saved.' +
' Error code: ' + LogCodes.ERR_FILEHAND + '.', e )
else:
Logger.get_instance().debug( 'FileHandlerUtil.save_obj_to_file(): ' +
str( len( objects_to_save ) ) +
' objects have been successfully saved in ' + file_path +
'. This file may be used to recover data later.' +
' Please see the documentation for more information.')
def pandas_df_to_csv( output_folder, filename, df, file_desc='', sep=',', ext='.csv', \
hdr=True, idx=False, mode='w', encoding='utf-8' ):
# Create the output folder if it does not yet exist
# (and its parent folders if necessary)
if not os.path.isdir( output_folder ):
os.makedirs( output_folder )
file_path = os.path.join( output_folder, filename ) + ext
Logger.get_instance().debug( 'FileHandlerUtil.pandas_df_to_csv(): ' +
' The data frame (' + file_desc + ') will be saved in ' +
file_path + '.' )
# Save the objects in the file
try:
df.to_csv( file_path,
sep = sep,
header = hdr,
index = idx,
mode = mode,
encoding = encoding )
except Exception as e:
raise DenCellORFException( 'FileHandlerUtil.pandas_df_to_csv(): An error occurred trying to' +
' save the pandas dataframe in ' + file_path +
' Error code: ' + LogCodes.ERR_FILEHAND + '.', e )
else:
Logger.get_instance().debug( 'FileHandlerUtil.pandas_df_to_csv(): ' +
' The data frame (' + file_desc +
') has been successfully saved in ' + file_path + '.' )
def add_and_commit(self, objects_to_add, process='Undefined process'):
# Get the number of objects that are expected to be inserted in the database
total_count = len(objects_to_add)
# Add the objects to the session
try:
self.get_session().add_all(objects_to_add)
except Exception as e:
# Get the number of objects of each type in the list
types_dict = GeneralUtil.get_type_counts_in_list(objects_to_add)
types_dict_str = ', '.join([
str(tp) + ': ' + str(val) for (tp, val) in types_dict.items()
])
raise DenCellORFException(
self.classname + '.add_and_commit():' +
' An error occurred trying to add ' + str(total_count) +
' objects (from ' + process + ') to the session.' +
' The list was containing the following objects: ' +
types_dict_str + '.', e)
# Commit changes
try:
self.commit()
except Exception as e:
# Get the number of objects of each type in the list
types_dict = GeneralUtil.get_type_counts_in_list(objects_to_add)
types_dict_str = ', '.join([
str(tp) + ': ' + str(val) for (tp, val) in types_dict.items()
])
raise DenCellORFException(
self.classname + '.add_and_commit():' +
' An error occurred trying to commit changes after addition of '
+ str(total_count) + ' objects (from ' + process +
') to the session.' +
' The list was containing the following objects: ' +
types_dict_str + '.', e)
# Log in debug mode the number of objects successfully inserted
Logger.get_instance().debug(
self.classname + '.add_and_commit(): ' + str(total_count) +
' objects (from ' + process +
') have been successfully added to the database.')
def initialize(self):
# Get the main keyword that defines the strategy
self.strategy = sys.argv[1]
# If the strategy is not known, check if the user asked the help.
# Otherwise, raise a DenCellORFException.
if (self.strategy not in OptionConstants.STRATEGIES_LIST):
# Display help on the console if necessary and exit the program
if (self.strategy in ['-h', '--help']):
print(
'To run a strategy, you need to type the command such as: \n'
+
'python $PYTHONPATH/fr/tagc/uorf/uorf.py [StrategyKeyword] [Options] \n'
+ 'or DenCellORF [StrategyKeyword] [Options]. \n'
'The following strategies are available: ' +
', '.join(OptionConstants.STRATEGIES_LIST) + '.\n'
' You may find more information about the options available for each strategy'
+ ' using the command DenCellORF [StrategyKeyword] -h' +
' or DenCellORF [StrategyKeyword] --help. \n' +
' For extensive information, please read the user manual (PDF file).'
)
exit()
else:
raise DenCellORFException(
'The strategy selected (' + self.strategy +
') is not correct.' + ' It must be one of ' +
', '.join(OptionConstants.STRATEGIES_LIST) +
'. Please see the documentation for more information.')
Logger.get_instance().info('---')
Logger.get_instance().info('Selected strategy: ' + self.strategy)
# Build an option parser to collect the option values
self.optionParser = OptionParser()
for current_prop_list in OptionConstants.OPTION_LIST[self.strategy]:
self.optionParser.add_option(current_prop_list[0],
current_prop_list[1],
action=current_prop_list[2],
type=current_prop_list[3],
dest=current_prop_list[4],
default=current_prop_list[5],
help=current_prop_list[6])
# Get the various option values into a dictionary
(opts, args) = self.optionParser.parse_args()
self.optionDict = vars(opts)
self.args = args
# Log the settings
Logger.get_instance().info('Settings:')
for opt in self.optionDict.items():
Logger.get_instance().info("-" + str(opt[0]) + ": '" +
str(opt[1]) + "'")
Logger.get_instance().info('---')
def list_to_string(list_to_convert, sep=',', not_none=False):
if (list_to_convert == None):
if not_none:
raise DenCellORFException(
'GeneralUtil.list_to_string(): None has been provided' +
' instead of a list.')
else:
return None
try:
list_from_str = sep.join(map(str, list_to_convert))
except Exception as e:
raise DenCellORFException(
'GeneralUtil.list_to_string(): An error occurred trying to convert'
+ ' the list into a string.', e)
else:
return list_from_str
def commit(self):
try:
self.get_session().commit()
except exc.SQLAlchemyError as e:
self.rollback_session()
raise DenCellORFException(
self.classname +
'.commit(): A SQLAlchemy error occurred trying' +
' to commit the session. Hence the session has been roll backed.',
e)
except Exception as e:
self.rollback_session()
raise DenCellORFException(
self.classname + '.commit(): An error occurred trying' +
' to commit the session. Hence the session has been roll backed.',
e)
finally:
self.close_session()
def find_sqce_consensus( list_of_sequences, sqce_type=Constants.SEQUENCE_TYPE_DNA, \
threshold=Constants.DEFAULT_SQCE_CONSENSUS_AMBIG_THRESHOLD, \
fasta_end_name = '' ):
if (sqce_type == Constants.SEQUENCE_TYPE_DNA):
alphabet = generic_dna
ambiguous = Constants.SEQUENCE_AMBIGUOUS_DNA_BASE
elif (sqce_type == Constants.SEQUENCE_TYPE_PROT):
alphabet = generic_protein
ambiguous = Constants.SEQUENCE_AMBIGUOUS_PROT_AA
else:
raise DenCellORFException(
'MergeStrategy.find_sqce_consensus(): The type of sequence provided'
+ ' has to be ' + Constants.SEQUENCE_TYPE_DNA + ' or ' +
Constants.SEQUENCE_TYPE_PROT + ' (provided type: ' +
str(sqce_type) + ').')
# Store the input sequences in a fasta file in order to run Muscle
input_sequences = (SeqRecord(Seq(s, alphabet))
for s in list_of_sequences)
if (not os.path.exists(DefaultTemporaryFolder.TEMPORARY_FOLDER)):
os.makedirs(DefaultTemporaryFolder.TEMPORARY_FOLDER)
input_sequences_file = os.path.join(
DefaultTemporaryFolder.TEMPORARY_FOLDER,
'input_sequences' + fasta_end_name + '.fasta')
SeqIO.write(input_sequences, input_sequences_file, 'fasta')
# Perform the multiple sequences alignment and
# store the output in a fasta file
aligned_sequences_file = os.path.join(
DefaultTemporaryFolder.TEMPORARY_FOLDER,
'aligned_sequences' + fasta_end_name + '.fasta')
muscle_cline = MuscleCommandline(cmd='/bin/muscle',
input=input_sequences_file,
out=aligned_sequences_file)
(stdout, stderr) = muscle_cline()
# Read the fasta file containing aligned sequences
align = AlignIO.read(aligned_sequences_file, 'fasta')
summary_align = AlignInfo.SummaryInfo(align)
# Compute the consensus
consensus = summary_align.gap_consensus(threshold=threshold,
ambiguous=ambiguous)
# Remove the temporary fasta files
os.remove(input_sequences_file)
os.remove(aligned_sequences_file)
return str(consensus)
def __init__(self):
# Get the options necessary to establish the connection to the database
self.db_settings = {}
self.db_settings[
Constants.DB_SETTINGS_DB_TYPE] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_TYPE)
self.db_settings[
Constants.DB_SETTINGS_DB_NAME] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_NAME, not_none=True)
if (self.db_settings[Constants.DB_SETTINGS_DB_TYPE] ==
SQLConstants.DB_TYPE_MYSQL):
self.db_settings[
Constants.DB_SETTINGS_MYSQL_USER] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_USER)
self.db_settings[
Constants.
DB_SETTINGS_MYSQL_PASSWD] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_PASSWD)
self.db_settings[
Constants.DB_SETTINGS_MYSQL_HOST] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_HOST_IP)
self.db_settings[
Constants.DB_SETTINGS_MYSQL_PORT] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_PORT)
elif (self.db_settings[Constants.DB_SETTINGS_DB_TYPE] ==
SQLConstants.DB_TYPE_SQLITE):
self.db_settings[
Constants.DB_SETTINGS_DB_FOLDER] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_FOLDER)
# Get the model of database
self.db_model = OptionManager.get_instance().get_option(
OptionConstants.OPTION_DATABASE_MODEL, not_none=True)
if (self.db_model not in OptionConstants.AVAILABLE_DATABASE_MODELS):
raise DenCellORFException(
'The database model provided has to be in the following list: '
+ ', '.join(OptionConstants.AVAILABLE_DATABASE_MODELS) + '.')
# Get the output folder
self.output_folder = OptionManager.get_instance().get_option(
OptionConstants.OPTION_OUTPUT_FOLDER, not_none=False)
if (not self.output_folder):
# By default, save the file in the default output folder
self.output_folder = Constants.DB_CONTENT_CONSISTENCY_ASSESSMENT_FOLDER
# Get the eventual name to use for the generated file
self.file_name = OptionManager.get_instance().get_option(
OptionConstants.OPTION_ASSESS_FILENAME, not_none=False)
if (not self.file_name):
# By default, save the file as 'DatabaseAssessment' file
# with the database model as prefix
self.file_name = self.db_model + 'DatabaseAssessment'
def dict_to_csv( output_folder, filename, dict, file_desc='', sort=False, sep=',', ext='.csv', \
hdr=None, key_func=lambda k: k, val_func=lambda v: v, unlist_key=False, unlist_val=False ):
# Create the output folder if it does not yet exist
# (and its parent folders if necessary)
if not os.path.isdir( output_folder ):
os.makedirs( output_folder )
file_path = os.path.join( output_folder, filename ) + ext
Logger.get_instance().debug( 'FileHandlerUtil.dict_to_csv(): The content of the dictionary (' +
file_desc + ') will be saved in ' + file_path + '.' )
# Save the objects in the file
try:
with open( file_path, 'wb' ) as csv_file:
writer = csv.writer( csv_file, delimiter = sep )
# Write the header if necessary
if hdr:
writer.writerow( hdr )
# Write the dictionary as key, value
if sort:
key_list = sorted( dict.keys() )
else:
key_list = dict.keys()
for k in key_list:
# Get the value and apply the functions to transform the key
# and value if necessary
key = key_func( k )
val = val_func( dict.get( k ) )
# Write the new row in the file
if ( ( not unlist_key ) or ( not isinstance( key, list ) ) ):
key = [ key ]
if ( ( not unlist_val ) or ( not isinstance( val, list ) ) ):
val = [ val ]
writer.writerow( key + val )
except Exception as e:
raise DenCellORFException( 'FileHandlerUtil.dict_to_csv(): An error occurred trying to save' +
' the content of the dictionary in ' + file_path +
'\n Error code: ' + LogCodes.ERR_FILEHAND + '.', e )
else:
Logger.get_instance().debug( 'FileHandlerUtil.dict_to_csv(): ' +
' the content of the dictionary (' + file_desc +
') has been successfully saved in ' + file_path +
'. Please see the documentation for more information.' )
def __init__(self):
# Get the options necessary to establish the connection to the database
self.db_settings = {}
self.db_settings[
Constants.DB_SETTINGS_DB_TYPE] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_TYPE)
self.db_settings[
Constants.DB_SETTINGS_DB_NAME] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_NAME, not_none=True)
if (self.db_settings[Constants.DB_SETTINGS_DB_TYPE] ==
SQLConstants.DB_TYPE_MYSQL):
self.db_settings[
Constants.DB_SETTINGS_MYSQL_USER] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_USER)
self.db_settings[
Constants.
DB_SETTINGS_MYSQL_PASSWD] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_PASSWD)
self.db_settings[
Constants.DB_SETTINGS_MYSQL_HOST] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_HOST_IP)
self.db_settings[
Constants.DB_SETTINGS_MYSQL_PORT] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_MYSQL_PORT)
elif (self.db_settings[Constants.DB_SETTINGS_DB_TYPE] ==
SQLConstants.DB_TYPE_SQLITE):
self.db_settings[
Constants.DB_SETTINGS_DB_FOLDER] = OptionManager.get_instance(
).get_option(OptionConstants.OPTION_DB_FOLDER)
# Get the model of database
self.db_model = OptionManager.get_instance().get_option(
OptionConstants.OPTION_DATABASE_MODEL, not_none=True)
if (self.db_model not in OptionConstants.AVAILABLE_DATABASE_MODELS):
raise DenCellORFException(
'The database model provided has to be in the following list: '
+ ', '.join(OptionConstants.AVAILABLE_DATABASE_MODELS) + '.')
# Get the output folder
self.output_folder = OptionManager.get_instance().get_option(
OptionConstants.OPTION_OUTPUT_FOLDER, not_none=False)
if (not self.output_folder):
# By default, save the files in a PRO / DS subfolder of the backup default folder
self.output_folder = os.path.join(Constants.BACKUP_DATA_FOLDER,
self.db_model)
# Get the eventual prefix to add to the file names
self.file_prefix = OptionManager.get_instance().get_option(
OptionConstants.OPTION_FILE_PREFIX, not_none=False)
def set_option(self, option_name, option_value):
if (self.optionDict == None):
self.optionDict = {}
if (option_name != None) and (len(option_name) > 0):
self.optionDict[option_name] = option_value
else:
raise DenCellORFException(
'OptionManager.set_option(): Trying to pass None as a key' +
' of the option dictionary.' + '\n Warning code: ' +
LogCodes.WARN_PROG_NONE + '.')
def min_max_median_of_list(list_of_values, fct=None):
min_val = None
max_val = None
median_val = None
if ((list_of_values != None) and (len(list_of_values) > 0)):
if fct:
# Raise an exception if the provided function is not allowed
if (fct not in ['int', 'float']):
raise DenCellORFException(
'GeneralUtil.min_max_val_of_list(): the function' +
' provided to convert the element of the list (' +
str(fct) + ') is not one of those authorized.' +
" It has to be 'int' or 'float'.")
list_of_values = map(eval(fct), list_of_values)
elif isinstance(list_of_values[0], str):
raise DenCellORFException(
'GeneralUtil.min_max_val_of_list(): The provided list (' +
str(list_of_values) + ' contains strings.' +
' The elements of the list need to be provided as integer'
+
' or float, or the "fct" option of the method needs to be used.'
)
# Get the minimal value of the list
min_val = min(list_of_values)
# Get the maximal value of the list
max_val = max(list_of_values)
# Get the median of the list
median_val = median(list_of_values)
return (min_val, max_val, median_val)
def get_option(self, option_name, not_none=False):
if not self.optionDict:
if not_none:
raise DenCellORFException(
'No option has been provided in the command line, while the option: "'
+ option_name + '" is expected.' +
' Please see the documentation for more information about the'
+ ' options available.')
else:
return None
if (option_name in self.optionDict.keys()):
return self.optionDict[option_name]
else:
if not_none:
raise DenCellORFException(
'The option: "' + option_name +
'" has to be provided in the command line.' +
' Please see the documentation for more information.')
else:
return None
def set_mode(self, logging_mode=Constants.MODE_INFO):
# If the verbosity level is incorrect, raise a DenCellORFException.
if (logging_mode not in Constants.LOG_MODES.values()):
raise DenCellORFException(
'The provided level of verbosity is incorrect. It must be one of: '
+ ', '.join(Constants.LOG_MODES.keys()) +
'. Please see the documentation for more information.')
else:
self.mode = logging_mode
# Reset the level of logger and handlers
self.logg.setLevel(logging_mode)
for hdl in self.logg.handlers:
hdl.setLevel(logging_mode)
return None
def string_to_list(str_to_convert, fct=None, sep=','):
if (str_to_convert == None):
return None
if (isinstance(str_to_convert, basestring)):
str_to_convert = str(str_to_convert).split(sep)
if fct:
return map(eval(fct), str_to_convert)
else:
return str_to_convert
else:
raise DenCellORFException(
'GeneralUtil.string_to_list(): The object provided has to be' +
' a string to be converted (type of the provided object: ' +
str(type(str_to_convert)) + ').')
def remove_sqlite_db(self):
if os.path.exists(self.db_path):
try:
remove(self.db_path)
except Exception as e:
raise DenCellORFException('The database located at ' +
str(self.db_path) +
' cannot be deleted.')
else:
Logger.get_instance().info('The database file located at ' +
str(self.db_path) +
' has been deleted.')
else:
Logger.get_instance().error(
self.classname + '.remove_sqlite_db(): There is no file' +
' located at' + str(self.db_path) + '.' + ' Error code: ' +
LogCodes.ERR_SQL_FILE + '.',
ex=False)
def build_database(self,
db_settings,
species,
sp_mandatory=True,
force_overwrite=False):
# Store the settings necessary to establish the connection
self.set_db_settings(db_settings)
# Check that a species is provided
if ((sp_mandatory) and ((species == None) or (len(species) == 0))):
raise DenCellORFException(
self.classname +
'.set_db_settings(): A species needs to be provided!')
# Get the engine to dedicated database
self.create_engine()
# Check and / or remove the existing database if necessary
if (self.db_type == SQLConstants.DB_TYPE_SQLITE):
reset_model = self.build_sqlite_database(force_overwrite)
# Check and / or remove the existing database if necessary
# and / or create the database on the server if necessary
elif (self.db_type == SQLConstants.DB_TYPE_MYSQL):
reset_model = self.build_mysql_database(force_overwrite)
# Open a session
self.create_session()
# If the model does not yet exists, create all the required tables
if reset_model:
self.BASE.metadata.create_all(self.engine)
Logger.get_instance().info('The database ' + self.db_path +
' has been created.')
else:
Logger.get_instance().info('The database ' + self.db_path +
' will be used.')
self.session.close()
def execute(self):
# Set the connection to the database
self.get_sqlmanager_instance().set_db_settings(self.db_settings)
# Check the integrity of the database
str_ok = self.get_sqlmanager_instance().check_database_str_integrity()
if (not str_ok):
raise DenCellORFException(
'The schema of the database provided does not follow' +
' the expected model. Please make sure the provided model (' +
self.db_model + ') and the database (' +
self.db_settings[Constants.DB_SETTINGS_DB_NAME] +
') provided are the right ones.')
self.init_log_file()
# Check the consistency according to the model
if (self.db_model == OptionConstants.DATABASE_DECLARATIVE_DS):
self.assess_DS_consistency()
elif (self.db_model == OptionConstants.DATABASE_DECLARATIVE_PRO):
self.assess_PRO_consistency()
def set_db_settings(self, db_settings):
# Store the settings of the database
self.db_settings = db_settings
# Store the database name
self.db_name = db_settings[Constants.DB_SETTINGS_DB_NAME]
# Check that a database name is provided
if (self.db_name == None or len(self.db_name) == 0):
raise DenCellORFException(
self.classname +
'.set_db_settings(): A database name needs to be provided!')
# Set the database type to default if not provided
if ((Constants.DB_SETTINGS_DB_TYPE in self.db_settings.keys())
and (self.db_settings[Constants.DB_SETTINGS_DB_TYPE] != None)):
self.db_type = self.db_settings[Constants.DB_SETTINGS_DB_TYPE]
else:
self.db_type = SQLConstants.DEFAULT_DB_TYPE
Logger.get_instance().debug(
self.classname + '.set_db_settings(): As no type of database' +
' has been provided, the database type has been set to ' +
self.db_type + '.')
# Set all the parameters to default that are not provided to default
if (self.db_type == SQLConstants.DB_TYPE_SQLITE):
if ((Constants.DB_SETTINGS_DB_FOLDER
not in self.db_settings.keys()) or
(self.db_settings[Constants.DB_SETTINGS_DB_FOLDER] == None)):
self.db_settings[
Constants.
DB_SETTINGS_DB_FOLDER] = SQLConstants.DEFAULT_SQLITE_PATH
if (self.db_settings[Constants.DB_SETTINGS_DB_FOLDER][-1] != '/'):
self.db_settings[Constants.DB_SETTINGS_DB_FOLDER] += '/'
elif (self.db_type == SQLConstants.DB_TYPE_MYSQL):
if ((Constants.DB_SETTINGS_MYSQL_USER
not in self.db_settings.keys()) or
(self.db_settings[Constants.DB_SETTINGS_MYSQL_USER] == None)):
self.db_settings[
Constants.
DB_SETTINGS_MYSQL_USER] = SQLConstants.DEFAULT_MYSQL_USER_NAME
if ((Constants.DB_SETTINGS_MYSQL_PASSWD
not in self.db_settings.keys())
or (self.db_settings[Constants.DB_SETTINGS_MYSQL_PASSWD]
== None)):
self.db_settings[
Constants.
DB_SETTINGS_MYSQL_PASSWD] = SQLConstants.DEFAULT_MYSQL_USER_PASSWD
if ((Constants.DB_SETTINGS_MYSQL_HOST
not in self.db_settings.keys()) or
(self.db_settings[Constants.DB_SETTINGS_MYSQL_HOST] == None)):
self.db_settings[
Constants.
DB_SETTINGS_MYSQL_HOST] = SQLConstants.DEFAULT_HOST_IP
if ((Constants.DB_SETTINGS_MYSQL_PORT
not in self.db_settings.keys()) or
(self.db_settings[Constants.DB_SETTINGS_MYSQL_PORT] == None)):
self.db_settings[
Constants.
DB_SETTINGS_MYSQL_PORT] = SQLConstants.DEFAULT_MYSQL_PORT
# Create the database URL
self.create_db_url()
def create_engine(self):
if (self.db_type in SQLConstants.AUTORIZED_DB_TYPES):
if (self.db_url != None):
# Create the engine
if (self.db_type == SQLConstants.DB_TYPE_MYSQL):
engine = create_engine(self.db_url + '?charset=utf8mb4',
encoding='utf-8',
pool_pre_ping=True)
# Set the maximal allowed size for packet of MySQL
@event.listens_for(engine, 'connect')
def set_max_allowed_packet(dbapi_connection,
connection_record):
cursor = dbapi_connection.cursor()
cursor.execute(
'SET GLOBAL max_allowed_packet = ' +
str(SQLConstants.MYSQL_MAX_ALLOWED_PACKET))
cursor.close()
# Set the maximum size of strings get by concatenating objects during queries
@event.listens_for(engine, 'connect')
def set_group_concat_max_len(dbapi_connection,
connection_record):
cursor = dbapi_connection.cursor()
cursor.execute(
'SET group_concat_max_len = ' +
str(SQLConstants.MYSQL_GROUP_CONCAT_MAX_LEN))
cursor.close()
elif (self.db_type == SQLConstants.DB_TYPE_SQLITE):
engine = create_engine(self.db_url,
encoding='utf-8',
pool_pre_ping=True)
# Enforce foreign key constraints
@event.listens_for(engine, 'connect')
def set_sqlite_pragma(dbapi_connection, connection_record):
cursor = dbapi_connection.cursor()
cursor.execute('PRAGMA foreign_keys=ON')
cursor.close()
# Set the maximum memory allowed for the range optimizer
@event.listens_for(engine, 'connect')
def set_range_optimizer_max_mem_size(dbapi_connection,
connection_record):
cursor = dbapi_connection.cursor()
cursor.execute(
'SET GLOBAL range_optimizer_max_mem_size = ' +
str(SQLConstants.RANGE_OPTIMIZER_MAX_MEM_SIZE))
cursor.close()
else:
engine = None
else:
raise DenCellORFException(
'SQLManger.create_engine(): the database type selected (' +
SQLConstants.DEFAULT_DB_TYPE + ') is not correct (accepted' +
' database types: ' +
', '.join(SQLConstants.AUTORIZED_DB_TYPES) +
'). Please see the documentation for more information.')
self.engine = engine
