def __init__(self, className: str, csvPath, cvsDelimiter,
functionnalUser: User, **kwargs):
self.logging = create_logger(className)
self.logging.debug("init...")
self.mongo_list = []
self.create_csv(csvPath=csvPath,
cvsDelimiter=cvsDelimiter,
functionnalUser=functionnalUser,
**kwargs)
self.logging.debug("done.")
def __init__(self, className, model, modelData, modelSql):
# Creation of the log
self.logging = create_logger(className)
self.logging.debug("starting...")
# Init value with input parameters
self.__model = model
self.__modelData = modelData
self.__sqlModel = modelSql
# Init variable
self.sqlObjectMap = dict()
self.logging.debug("done")
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User,
parent_list, order: int):
# Check input variable
if not parent_list:
self.logging.critical(
"No parent list impossible to continue !")
self.logging = create_logger("DepartmentData")
self.dep = fromCvsFile[0]
self.reg = to_integer(fromCvsFile[1])
self.cheflieu = str(fromCvsFile[2])
self.tncc = fromCvsFile[3]
self.ncc = fromCvsFile[4]
self.nccenr = fromCvsFile[5]
self.libelle = fromCvsFile[6]
# Check variable
# - dep
if not self.dep:
self.logging.critical(
f"Value '{self.dep}' is not a number !")
# - reg
if not self.reg:
self.logging.critical(
f"Value '{self.reg}' is not a number !")
# - nccenr
if not self.nccenr:
self.logging.critical(f"Value '{self.nccenr}' is empty !")
# - parent
l_parent = None
for parent in parent_list:
if parent.insee == self.reg:
l_parent = parent
if not parent:
self.logging.critical(
f"Parent not find with insee value '{self.reg}' !")
# Create Localisation
l_localisationForm = LocalisationForm(
data={
'code': self.dep,
'insee': int(
re.compile("(\d+)").match(self.dep).group(
1)), # required for 2A and 2B
'is_enable': True,
'is_linkeable': True,
'parent': l_parent,
'order': order
})
if l_localisationForm.is_valid() is False:
self.logging.critical(
f"Error on Localisation with code={self.codeiso3}: {l_localisationForm.errors}"
)
l_localisation = l_localisationForm.save(user=functionnalUser)
# Create LocalisationData (FR)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.nccenr,
'resume': 'Département Français',
'language': LanguageAvailable.FR.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Create LocalisationData (EN)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.nccenr,
'resume': 'French Department',
'language': LanguageAvailable.EN.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.libcog_en}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Register localisation
mongo_list.append(l_localisation)
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User,
france, order: int):
if not france:
self.critical(
f"France contains nothing impossible to continue !")
self.logging = create_logger("RegionData")
self.reg = to_integer(fromCvsFile[0])
self.cheflieu = fromCvsFile[1]
self.tncc = to_lowerCase(fromCvsFile[2])
self.ncc = to_lowerCase(fromCvsFile[3])
self.nccenr = to_lowerCase(fromCvsFile[4])
self.libelle = fromCvsFile[5]
# Check Variables
# - Reg
if not self.reg:
self.logging.error(
f"Value '{self.cog}' is not a number ! Ignore it...")
return
# Create Localisation
l_localisationForm = LocalisationForm(
data={
'code': self.cheflieu,
'insee': self.reg,
'is_enable': True,
'is_linkeable': True,
'parent': france,
'order': order
})
if l_localisationForm.is_valid() is False:
self.logging.critical(
f"Error on Localisation with code={self.cheflieu}: {l_localisationForm.errors}"
)
l_localisation = l_localisationForm.save(user=functionnalUser)
# Create LocalisationData (FR)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.nccenr,
'resume': 'Région Française',
'localisation': l_localisation,
'language': LanguageAvailable.FR.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.cheflieu}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Create LocalisationData (EN)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.nccenr,
'resume': 'French Region',
'localisation': l_localisation,
'language': LanguageAvailable.EN.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.cheflieu}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Register localisation
mongo_list.append(l_localisation)
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User,
order: int, parent_list):
# Check input variable
if not parent_list:
self.logging.critical(
"No parent list impossible to continue !")
self.logging = create_logger("CountryData")
self.cog = to_integer(fromCvsFile[0])
self.actual = to_integer(fromCvsFile[1])
self.capay = fromCvsFile[2]
self.crpay = fromCvsFile[3]
self.ani = fromCvsFile[4]
self.libcog = to_lowerCase(fromCvsFile[5])
self.libenr = fromCvsFile[6]
self.ancnom = fromCvsFile[7]
self.codeiso2 = fromCvsFile[8]
self.codeiso3 = fromCvsFile[9]
self.codenum3 = fromCvsFile[10]
# To Ignore:
if self.actual != 1:
return
# Check value
if self.cog is None:
if self.libcog != "France":
self.logging.error(
f"Value '{self.cog}' is not a number ! Ignore it..."
)
return
self.cog = 99100
if self.cog < self.cog_minimum_value:
self.logging.warning(
f"cog value \'{self.cog}\' is less than {self.cog_minimum_value} ! Ignore it..."
)
return
if not self.libcog:
self.logging.error(
f"Value '{self.libcog}' is empty after regexp ! Ignore it..."
)
return
# - parent
l_parent = None
l_parent_insee = int(str(self.cog)[:3]) * 100
for parent in parent_list:
if parent.insee == l_parent_insee:
l_parent = parent
if not l_parent:
self.logging.warning(
f"Parent not find with insee value \'{l_parent_insee}\' ! Ignore it..."
)
return
# Create Localisation
l_localisationForm = LocalisationForm(
data={
'code': self.codeiso3,
'insee': self.cog,
'is_enable': True,
'is_linkeable': True,
'parent': l_parent,
'order': order
})
if l_localisationForm.is_valid() is False:
self.logging.critical(
f"Error on Localisation with code={self.libcog}: {l_localisationForm.errors}"
)
l_localisation = l_localisationForm.save(user=functionnalUser)
# Create LocalisationData (FR)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.libcog,
'resume': 'Pays',
'localisation': l_localisation,
'language': LanguageAvailable.FR.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Create LocalisationData (EN)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.libcog,
'resume': 'Country',
'localisation': l_localisation,
'language': LanguageAvailable.EN.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Register localisation
mongo_list.append(l_localisation)
def __init__(self, fromCvsFile, mongo_list, functionnalUser: User,
order: int):
self.logging = create_logger("ContinentData")
self.cog = to_integer(fromCvsFile[0])
self.libcog = to_lowerCase(fromCvsFile[1])
self.libcog_en = to_lowerCase(fromCvsFile[2])
self.codeiso2 = fromCvsFile[3]
self.codeiso3 = fromCvsFile[4]
# Check Variable
if not self.cog:
self.logging.error(
f"Value '{self.cog}' is not a number ! Ignore it...")
return
if not self.libcog:
self.logging.error(
f"Value '{self.libcog}' is empty after regexp ! Ignore it..."
)
return
if not self.libcog_en:
self.logging.error(
f"Value '{self.libcog_en}' is empty after regexp ! Ignore it..."
)
return
# Create Localisation
l_localisationForm = LocalisationForm(
data={
'code': self.codeiso3,
'insee': self.cog,
'is_enable': True,
'is_linkeable': True,
'parent': None,
'order': order
})
if l_localisationForm.is_valid() is False:
self.logging.critical(
f"Error on Localisation with code={self.codeiso3}: {l_localisationForm.errors}"
)
l_localisation = l_localisationForm.save(user=functionnalUser)
# Create LocalisationData (FR)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.libcog,
'resume': 'Continent',
'language': LanguageAvailable.FR.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.libcog}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Create LocalisationData (EN)
l_localisationDataForm = LocalisationDataForm(
data={
'name': self.libcog_en,
'resume': 'Continent',
'language': LanguageAvailable.EN.value
})
if l_localisationDataForm.is_valid(
localisation=l_localisation) is False:
self.logging.critical(
f"Error on LocalisationData with name={self.libcog_en}: {l_localisationDataForm.errors}"
)
l_localisationDataForm.save()
# Register localisation
mongo_list.append(l_localisation)
def run():
# -----------------------------------
# Logger
# -----------------------------------
logging = create_logger('sqlImport', deletePreviousLog=True)
logging.info("Start SQL importation...")
# -----------------------------------
# Read config file for SQL
# -----------------------------------
l_sqlConfig_filename = "config.ini"
l_sqlConfig_folder = "/etc"
l_sqlConfig_file = f"{l_sqlConfig_folder}/{l_sqlConfig_filename}"
import configparser
settings = configparser.ConfigParser()
settings._interpolation = configparser.ExtendedInterpolation()
if not settings.read(os.path.join(os.path.abspath(os.path.dirname(__file__)), f"{l_sqlConfig_file}")):
logging.critical(f"main - Impossible to open {l_sqlConfig_file} !")
logging.info(f"File {l_sqlConfig_file} has been load !")
# -----------------------------------
# Connexion to SQL Database
# -----------------------------------
NetLiensSqlNetwork = SqlConnection(settings.get('sql_netLiens', 'address'),
settings.get('sql_netLiens', 'user'),
settings.get('sql_netLiens', 'password'),
settings.get('sql_netLiens', 'databasename'))
logging.info(f"SQL Database has been set !")
# # -----------------------------------
# # Start conversion
# # -----------------------------------
logging.info("*********************************")
logging.info("Conversion starting...")
logging.info("Get SQL Data...")
# l_annuCat_map = readSqlTable(
# logging=logging,
# sqlConnection=NetLiensSqlNetwork,
# sql_table_name=CategoryManager.sqlTableName,
# order='cat_parent'
# )
# l_annuDept_map = readSqlTable(
# logging=logging,
# sqlConnection=NetLiensSqlNetwork,
# sql_table_name=LocalisationManager.sqlTableName
# )
l_annuSite_map = readSqlTable(
logging=logging,
sqlConnection=NetLiensSqlNetwork,
sql_table_name=AnnouncementManager.sqlTableName
)
logging.info("Starting Management instances...")
l_category = CategoryManager()
l_localisation = LocalisationManager()
l_announcement = AnnouncementManager()
logging.info("Delete database...")
l_category.deleteModel()
l_localisation.deleteModel()
l_announcement.deleteModel()
User.objects.all().delete()
logging.info("Creation of functionnal user...")
l_functionnalUser = User.objects.create_superuser(
email=settings.get('import_username', 'email'),
password=settings.get('import_username', 'password')
)
logging.info("Category starting...")
# l_category.createSqlObject(
# sqlObjectMap = l_annuCat_map,
# functionnalUser = l_functionnalUser
# )
# l_category.createModelsFromSqlObjectMap(
# functionnalUser = l_functionnalUser
# )
logging.info("Category conversion [OK]")
logging.info("Localisation Conversion starting...")
# l_localisation.createModelsFromInseeFile(
# sqlObjectMap = l_annuDept_map,
# functionnalUser = l_functionnalUser
# )
logging.info("Localisation conversion [OK]")
logging.info("Announcement Conversion starting...")
l_announcement.createSqlObject(
sqlObjectMap=l_annuSite_map,
functionnalUser=l_functionnalUser
)
l_announcement.createModelsFromSqlObjectMap(
functionnalUser = l_functionnalUser
)
logging.info("Announcement conversion [OK]")
logging.info("Conversion [OK]")
logging.info("*********************************\n")