def load_pays():
load = Loader(
"https://simplonline-v3-prod.s3.eu-west-3.amazonaws.com/media/file/csv/25d9c746-3622-4c48-835e-d7ccafa311f5.csv"
, "../datas/RAW/"
)
csv_path = load.ensure_data_loaded()
pec.clean_csv(
'../datas/RAW/' + csv_path,
"../datas/CURATED/pays_en_chiffre.csv")
pec.jsonify_csv(
"../datas/CURATED/pays_en_chiffre.csv",
"../datas/CURATED/pays_en_chiffre.json"
)
pays: dict = json.load(open("../datas/CURATED/pays_en_chiffre.json"))
if mongo.db.pays:
mongo.db.pays.drop()
db_pays = mongo.db["pays"]
db_pays.insert_many(pays)
return jsonify(
etat="success"
)
def train(labels_array, nb_epochs, nb_patience):
import tensorflow as tf
from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Conv2D, GlobalAveragePooling2D, GlobalMaxPooling2D, MaxPooling2D, AveragePooling2D, Activation, Dropout, Flatten, Dense
from tensorflow.keras.callbacks import EarlyStopping
from tensorflow.keras.preprocessing import image
import mlflow
import mlflow.tensorflow
# Telechargement du ZIP
from modules.loader import Loader
loader = Loader(
"https://stdatalake010.blob.core.windows.net/public/cifar-100.zip",
'../datas/ZIP/',
extraction_target='../datas/RAW/'
)
loader.ensure_data_loaded()
# Extraction du jeu de donnees
from modules.splitting import Splitting
labels_array = ['apple', 'bee']
TRAIN_DATA_DIR = Splitting.copie_dossiers(
'../datas/RAW/train',
labels_array,
500,
explorer=False
)
print(TRAIN_DATA_DIR)
# Chargement des images
image_data_generator = tf.keras.preprocessing.image.ImageDataGenerator(validation_split=0.2)
TRAIN_IMAGE_SIZE = 32
TRAIN_BATCH_SIZE = 64
train_generator = image_data_generator.flow_from_directory(
TRAIN_DATA_DIR,
target_size=(TRAIN_IMAGE_SIZE, TRAIN_IMAGE_SIZE),
batch_size=TRAIN_BATCH_SIZE,
class_mode='categorical',
subset='training')
validation_generator = image_data_generator.flow_from_directory(
TRAIN_DATA_DIR, # same directory as training data
target_size=(TRAIN_IMAGE_SIZE, TRAIN_IMAGE_SIZE),
batch_size=TRAIN_BATCH_SIZE,
class_mode='categorical',
subset='validation')
with mlflow.start_run():
model = Sequential()
model.add(Conv2D(32, kernel_size=3, activation='elu', kernel_initializer='he_uniform', padding='same', input_shape=(32,32,3)))
#Toujours à la fin
model.add(Flatten())
model.add(Dense(2, activation='softmax'))
model.compile(optimizer='adam', loss='categorical_crossentropy', metrics=['accuracy'])
es_callback = EarlyStopping(monitor='val_loss', patience=nb_patience)
training = model.fit(train_generator, epochs=nb_epochs, callbacks=[es_callback], validation_data=validation_generator, shuffle=False)