def __init__(self, className: str, csvPath, cvsDelimiter, functionnalUser: User, **kwargs): self.logging = create_logger(className) self.logging.debug("init...") self.mongo_list = [] self.create_csv(csvPath=csvPath, cvsDelimiter=cvsDelimiter, functionnalUser=functionnalUser, **kwargs) self.logging.debug("done.")
def __init__(self, className, model, modelData, modelSql): # Creation of the log self.logging = create_logger(className) self.logging.debug("starting...") # Init value with input parameters self.__model = model self.__modelData = modelData self.__sqlModel = modelSql # Init variable self.sqlObjectMap = dict() self.logging.debug("done")
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User, parent_list, order: int): # Check input variable if not parent_list: self.logging.critical( "No parent list impossible to continue !") self.logging = create_logger("DepartmentData") self.dep = fromCvsFile[0] self.reg = to_integer(fromCvsFile[1]) self.cheflieu = str(fromCvsFile[2]) self.tncc = fromCvsFile[3] self.ncc = fromCvsFile[4] self.nccenr = fromCvsFile[5] self.libelle = fromCvsFile[6] # Check variable # - dep if not self.dep: self.logging.critical( f"Value '{self.dep}' is not a number !") # - reg if not self.reg: self.logging.critical( f"Value '{self.reg}' is not a number !") # - nccenr if not self.nccenr: self.logging.critical(f"Value '{self.nccenr}' is empty !") # - parent l_parent = None for parent in parent_list: if parent.insee == self.reg: l_parent = parent if not parent: self.logging.critical( f"Parent not find with insee value '{self.reg}' !") # Create Localisation l_localisationForm = LocalisationForm( data={ 'code': self.dep, 'insee': int( re.compile("(\d+)").match(self.dep).group( 1)), # required for 2A and 2B 'is_enable': True, 'is_linkeable': True, 'parent': l_parent, 'order': order }) if l_localisationForm.is_valid() is False: self.logging.critical( f"Error on Localisation with code={self.codeiso3}: {l_localisationForm.errors}" ) l_localisation = l_localisationForm.save(user=functionnalUser) # Create LocalisationData (FR) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.nccenr, 'resume': 'Département Français', 'language': LanguageAvailable.FR.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Create LocalisationData (EN) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.nccenr, 'resume': 'French Department', 'language': LanguageAvailable.EN.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.libcog_en}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Register localisation mongo_list.append(l_localisation)
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User, france, order: int): if not france: self.critical( f"France contains nothing impossible to continue !") self.logging = create_logger("RegionData") self.reg = to_integer(fromCvsFile[0]) self.cheflieu = fromCvsFile[1] self.tncc = to_lowerCase(fromCvsFile[2]) self.ncc = to_lowerCase(fromCvsFile[3]) self.nccenr = to_lowerCase(fromCvsFile[4]) self.libelle = fromCvsFile[5] # Check Variables # - Reg if not self.reg: self.logging.error( f"Value '{self.cog}' is not a number ! Ignore it...") return # Create Localisation l_localisationForm = LocalisationForm( data={ 'code': self.cheflieu, 'insee': self.reg, 'is_enable': True, 'is_linkeable': True, 'parent': france, 'order': order }) if l_localisationForm.is_valid() is False: self.logging.critical( f"Error on Localisation with code={self.cheflieu}: {l_localisationForm.errors}" ) l_localisation = l_localisationForm.save(user=functionnalUser) # Create LocalisationData (FR) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.nccenr, 'resume': 'Région Française', 'localisation': l_localisation, 'language': LanguageAvailable.FR.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.cheflieu}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Create LocalisationData (EN) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.nccenr, 'resume': 'French Region', 'localisation': l_localisation, 'language': LanguageAvailable.EN.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.cheflieu}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Register localisation mongo_list.append(l_localisation)
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User, order: int, parent_list): # Check input variable if not parent_list: self.logging.critical( "No parent list impossible to continue !") self.logging = create_logger("CountryData") self.cog = to_integer(fromCvsFile[0]) self.actual = to_integer(fromCvsFile[1]) self.capay = fromCvsFile[2] self.crpay = fromCvsFile[3] self.ani = fromCvsFile[4] self.libcog = to_lowerCase(fromCvsFile[5]) self.libenr = fromCvsFile[6] self.ancnom = fromCvsFile[7] self.codeiso2 = fromCvsFile[8] self.codeiso3 = fromCvsFile[9] self.codenum3 = fromCvsFile[10] # To Ignore: if self.actual != 1: return # Check value if self.cog is None: if self.libcog != "France": self.logging.error( f"Value '{self.cog}' is not a number ! Ignore it..." ) return self.cog = 99100 if self.cog < self.cog_minimum_value: self.logging.warning( f"cog value \'{self.cog}\' is less than {self.cog_minimum_value} ! Ignore it..." ) return if not self.libcog: self.logging.error( f"Value '{self.libcog}' is empty after regexp ! Ignore it..." ) return # - parent l_parent = None l_parent_insee = int(str(self.cog)[:3]) * 100 for parent in parent_list: if parent.insee == l_parent_insee: l_parent = parent if not l_parent: self.logging.warning( f"Parent not find with insee value \'{l_parent_insee}\' ! Ignore it..." ) return # Create Localisation l_localisationForm = LocalisationForm( data={ 'code': self.codeiso3, 'insee': self.cog, 'is_enable': True, 'is_linkeable': True, 'parent': l_parent, 'order': order }) if l_localisationForm.is_valid() is False: self.logging.critical( f"Error on Localisation with code={self.libcog}: {l_localisationForm.errors}" ) l_localisation = l_localisationForm.save(user=functionnalUser) # Create LocalisationData (FR) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.libcog, 'resume': 'Pays', 'localisation': l_localisation, 'language': LanguageAvailable.FR.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Create LocalisationData (EN) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.libcog, 'resume': 'Country', 'localisation': l_localisation, 'language': LanguageAvailable.EN.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Register localisation mongo_list.append(l_localisation)
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User, order: int): self.logging = create_logger("ContinentData") self.cog = to_integer(fromCvsFile[0]) self.libcog = to_lowerCase(fromCvsFile[1]) self.libcog_en = to_lowerCase(fromCvsFile[2]) self.codeiso2 = fromCvsFile[3] self.codeiso3 = fromCvsFile[4] # Check Variable if not self.cog: self.logging.error( f"Value '{self.cog}' is not a number ! Ignore it...") return if not self.libcog: self.logging.error( f"Value '{self.libcog}' is empty after regexp ! Ignore it..." ) return if not self.libcog_en: self.logging.error( f"Value '{self.libcog_en}' is empty after regexp ! Ignore it..." ) return # Create Localisation l_localisationForm = LocalisationForm( data={ 'code': self.codeiso3, 'insee': self.cog, 'is_enable': True, 'is_linkeable': True, 'parent': None, 'order': order }) if l_localisationForm.is_valid() is False: self.logging.critical( f"Error on Localisation with code={self.codeiso3}: {l_localisationForm.errors}" ) l_localisation = l_localisationForm.save(user=functionnalUser) # Create LocalisationData (FR) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.libcog, 'resume': 'Continent', 'language': LanguageAvailable.FR.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Create LocalisationData (EN) l_localisationDataForm = LocalisationDataForm( data={ 'name': self.libcog_en, 'resume': 'Continent', 'language': LanguageAvailable.EN.value }) if l_localisationDataForm.is_valid( localisation=l_localisation) is False: self.logging.critical( f"Error on LocalisationData with name={self.libcog_en}: {l_localisationDataForm.errors}" ) l_localisationDataForm.save() # Register localisation mongo_list.append(l_localisation)
def run(): # ----------------------------------- # Logger # ----------------------------------- logging = create_logger('sqlImport', deletePreviousLog=True) logging.info("Start SQL importation...") # ----------------------------------- # Read config file for SQL # ----------------------------------- l_sqlConfig_filename = "config.ini" l_sqlConfig_folder = "/etc" l_sqlConfig_file = f"{l_sqlConfig_folder}/{l_sqlConfig_filename}" import configparser settings = configparser.ConfigParser() settings._interpolation = configparser.ExtendedInterpolation() if not settings.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), f"{l_sqlConfig_file}")): logging.critical(f"main - Impossible to open {l_sqlConfig_file} !") logging.info(f"File {l_sqlConfig_file} has been load !") # ----------------------------------- # Connexion to SQL Database # ----------------------------------- NetLiensSqlNetwork = SqlConnection(settings.get('sql_netLiens', 'address'), settings.get('sql_netLiens', 'user'), settings.get('sql_netLiens', 'password'), settings.get('sql_netLiens', 'databasename')) logging.info(f"SQL Database has been set !") # # ----------------------------------- # # Start conversion # # ----------------------------------- logging.info("*********************************") logging.info("Conversion starting...") logging.info("Get SQL Data...") # l_annuCat_map = readSqlTable( # logging=logging, # sqlConnection=NetLiensSqlNetwork, # sql_table_name=CategoryManager.sqlTableName, # order='cat_parent' # ) # l_annuDept_map = readSqlTable( # logging=logging, # sqlConnection=NetLiensSqlNetwork, # sql_table_name=LocalisationManager.sqlTableName # ) l_annuSite_map = readSqlTable( logging=logging, sqlConnection=NetLiensSqlNetwork, sql_table_name=AnnouncementManager.sqlTableName ) logging.info("Starting Management instances...") l_category = CategoryManager() l_localisation = LocalisationManager() l_announcement = AnnouncementManager() logging.info("Delete database...") l_category.deleteModel() l_localisation.deleteModel() l_announcement.deleteModel() User.objects.all().delete() logging.info("Creation of functionnal user...") l_functionnalUser = User.objects.create_superuser( email=settings.get('import_username', 'email'), password=settings.get('import_username', 'password') ) logging.info("Category starting...") # l_category.createSqlObject( # sqlObjectMap = l_annuCat_map, # functionnalUser = l_functionnalUser # ) # l_category.createModelsFromSqlObjectMap( # functionnalUser = l_functionnalUser # ) logging.info("Category conversion [OK]") logging.info("Localisation Conversion starting...") # l_localisation.createModelsFromInseeFile( # sqlObjectMap = l_annuDept_map, # functionnalUser = l_functionnalUser # ) logging.info("Localisation conversion [OK]") logging.info("Announcement Conversion starting...") l_announcement.createSqlObject( sqlObjectMap=l_annuSite_map, functionnalUser=l_functionnalUser ) l_announcement.createModelsFromSqlObjectMap( functionnalUser = l_functionnalUser ) logging.info("Announcement conversion [OK]") logging.info("Conversion [OK]") logging.info("*********************************\n")