class dataTransformPredict: """ This class shall be used for transforming the Good Raw Training Data before loading it in Database!!. """ def __init__(self): self.goodDataPath = "Prediction_Good_Raw_Files_Validated" self.logger = App_Logger() self.awsObj = AwsStorageManagement() def addQuotesToStringValuesInColumn(self): """ Method Name: addQuotesToStringValuesInColumn Description: This method replaces the missing values in columns with "NULL" to store in the table. We are using substring in the first column to keep only "Integer" data for ease up the loading. This column is anyways going to be removed during prediction. """ try: log_file = 'dataTransformLog' onlyfiles = self.awsObj.listDirFiles(self.goodDataPath) for file in onlyfiles: data = self.awsObj.csvToDataframe(self.goodDataPath, file) data['stalk-root'] = data['stalk-root'].replace('?', "'?'") self.awsObj.saveDataframeToCsv(self.goodDataPath, file, data) self.logger.log(log_file, " %s: Quotes added successfully!!" % file) except Exception as e: log_file = 'dataTransformLog' self.logger.log(log_file, "Data Transformation failed because:: %s" % e) raise e
class dataTransform: """ This class shall be used for transforming the Good Raw Training Data before loading it in Database!!. """ def __init__(self): self.goodDataPath = "Training_Good_Raw_Files_Validated" self.logger = App_Logger() self.awsObj = AwsStorageManagement() def addQuotesToStringValuesInColumn(self): """ Method Name: addQuotesToStringValuesInColumn Description: This method converts all the columns with string datatype such that each value for that column is enclosed in quotes. This is done to avoid the error while inserting string values in table as varchar. """ log_file = 'addQuotesToStringValuesInColumn' try: onlyfiles = self.awsObj.listDirFiles(self.goodDataPath) for file in onlyfiles: data = self.awsObj.csvToDataframe(self.goodDataPath, file) for column in data.columns: count = data[column][data[column] == '?'].count() if count != 0: data[column] = data[column].replace('?', "'?'") self.awsObj.saveDataframeToCsv(self.goodDataPath, file, data) self.logger.log(log_file," %s: Quotes added successfully!!" % file) except Exception as e: self.logger.log(log_file, "Data Transformation failed because:: %s" % e)
class Prediction_Data_validation: """ This class shall be used for handling all the validation done on the Raw Prediction Data!!. """ def __init__(self, path): self.Batch_Directory = path self.schema_path = 'schema_prediction.json' self.logger = App_Logger() self.awsObj = AwsStorageManagement() self.dbObj = mongoDBOperation() def valuesFromSchema(self): """ Method Name: valuesFromSchema Description: This method extracts all the relevant information from the pre-defined "Schema" file. Output: LengthOfDateStampInFile, LengthOfTimeStampInFile, column_names, Number of Columns On Failure: Raise ValueError,KeyError,Exception """ try: if not self.dbObj.isCollectionPresent('mushroomClassifierDB', 'predict_schema'): with open(self.schema_path, 'r') as f: dic = json.load(f) f.close() self.dbObj.insertOneRecord('mushroomClassifierDB', 'predict_schema', dic) dic = self.dbObj.getRecords('mushroomClassifierDB', 'predict_schema') pattern = dic['SampleFileName'] LengthOfDateStampInFile = dic['LengthOfDateStampInFile'] LengthOfTimeStampInFile = dic['LengthOfTimeStampInFile'] column_names = dic['ColName'] NumberofColumns = dic['NumberofColumns'] file = 'valuesfromSchemaValidationLog' message = "LengthOfDateStampInFile:: %s" % LengthOfDateStampInFile + "\t" + "LengthOfTimeStampInFile:: %s" % LengthOfTimeStampInFile + "\t " + "NumberofColumns:: %s" % NumberofColumns + "\n" self.logger.log(file, message) except ValueError: file = 'valuesfromSchemaValidationLog' self.logger.log( file, "ValueError:Value not found inside schema_training.json") raise ValueError except KeyError: file = 'valuesfromSchemaValidationLog' self.logger.log(file, "KeyError:Key value error incorrect key passed") raise KeyError except Exception as e: file = 'valuesfromSchemaValidationLog' self.logger.log(file, str(e)) raise e return LengthOfDateStampInFile, LengthOfTimeStampInFile, column_names, NumberofColumns def manualRegexCreation(self): """ Method Name: manualRegexCreation Description: This method contains a manually defined regex based on the "FileName" given in "Schema" file. This Regex is used to validate the filename of the prediction data. Output: Regex pattern On Failure: None """ regex = "['mushroom']+['\_'']+[\d_]+[\d]+\.csv" return regex def createDirectoryForGoodBadRawData(self): """ Method Name: createDirectoryForGoodBadRawData Description: This method creates directories to store the Good Data and Bad Data after validating the prediction data. Output: None On Failure: Exception """ try: self.awsObj.createS3Directory( 'Prediction_Good_Raw_Files_Validated') self.awsObj.createS3Directory('Prediction_Bad_Raw_Files_Validated') except Exception as ex: file = 'GeneralLog' self.logger.log(file, "Error while creating Directory %s:" % ex) def deleteExistingGoodDataTrainingFolder(self): """ Method Name: deleteExistingGoodDataTrainingFolder Description: This method deletes the directory made to store the Good Data after loading the data in the table. Once the good files are loaded in the DB,deleting the directory ensures space optimization. Output: None On Failure: Exception """ try: file = 'GeneralLog' self.logger.log(file, "GoodRaw directory deleted successfully!!!") self.awsObj.deleteDirectory('Prediction_Good_Raw_Files_Validated') except Exception as s: file = 'GeneralLog' self.logger.log(file, "Error while Deleting Directory : %s" % s) raise s def deleteExistingBadDataTrainingFolder(self): """ Method Name: deleteExistingBadDataTrainingFolder Description: This method deletes the directory made to store the bad Data. Output: None On Failure: Exception """ try: file = 'GeneralLog' self.logger.log( file, "BadRaw directory deleted before starting validation!!!") self.awsObj.deleteDirectory('Prediction_Bad_Raw_Files_Validated') except Exception as s: file = 'GeneralLog' self.logger.log(file, "Error while Deleting Directory : %s" % s) raise s def moveBadFilesToArchiveBad(self): """ Method Name: moveBadFilesToArchiveBad Description: This method deletes the directory made to store the Bad Data after moving the data in an archive folder. We archive the bad files to send them back to the client for invalid data issue. Output: None On Failure: Exception """ now = datetime.now() date = now.date() time = now.strftime("%H%M%S") try: target_folder = 'PredictionArchivedBadData/BadData_' + str( date) + "_" + str(time) self.awsObj.copyFileToFolder('Prediction_Bad_Raw_Files_Validated', target_folder) file = 'GeneralLog' self.logger.log(file, "Bad files moved to archive") self.logger.log(file, "Bad Raw Data Folder Deleted successfully!!") except Exception as e: file = 'GeneralLog' self.logger.log(file, "Error while moving bad files to archive:: %s" % e) raise e def validationFileNameRaw(self, regex, LengthOfDateStampInFile, LengthOfTimeStampInFile): """ Method Name: validationFileNameRaw Description: This function validates the name of the prediction csv file as per given name in the schema! Regex pattern is used to do the validation.If name format do not match the file is moved to Bad Raw Data folder else in Good raw data. Output: None On Failure: Exception """ # delete the directories for good and bad data in case last run was unsuccessful and folders were not deleted. self.deleteExistingBadDataTrainingFolder() self.deleteExistingGoodDataTrainingFolder() self.createDirectoryForGoodBadRawData() batch_dir = self.Batch_Directory.strip('/').strip('\\') print('Prediction File Path: ', batch_dir) self.awsObj.uploadFiles(batch_dir, batch_dir) onlyfiles = self.awsObj.listDirFiles(batch_dir) try: f = 'nameValidationLog' for filename in onlyfiles: if (re.match(regex, filename)): splitAtDot = re.split('.csv', filename) splitAtDot = (re.split('_', splitAtDot[0])) if len(splitAtDot[1]) == LengthOfDateStampInFile: if len(splitAtDot[2]) == LengthOfTimeStampInFile: self.awsObj.copyFileToFolder( batch_dir, 'Prediction_Good_Raw_Files_Validated', filename) self.logger.log( f, "Valid File name!! File moved to GoodRaw Folder :: %s" % filename) else: self.awsObj.copyFileToFolder( self.Batch_Directory, 'Prediction_Bad_Raw_Files_Validated', filename) self.logger.log( f, "Invalid File Name!! File moved to Bad Raw Folder :: %s" % filename) else: self.awsObj.copyFileToFolder( self.Batch_Directory, 'Prediction_Bad_Raw_Files_Validated', filename) self.logger.log( f, "Invalid File Name!! File moved to Bad Raw Folder :: %s" % filename) else: self.awsObj.copyFileToFolder( self.Batch_Directory, 'Prediction_Bad_Raw_Files_Validated', filename) self.logger.log( f, "Invalid File Name!! File moved to Bad Raw Folder :: %s" % filename) except Exception as e: f = 'nameValidationLog' self.logger.log(f, "Error occured while validating FileName %s" % e) raise e def validateColumnLength(self, NumberofColumns): """ Method Name: validateColumnLength Description: This function validates the number of columns in the csv files. It is should be same as given in the schema file. If not same file is not suitable for processing and thus is moved to Bad Raw Data folder. If the column number matches, file is kept in Good Raw Data for processing. The csv file is missing the first column name, this function changes the missing name to "Wafer". Output: None On Failure: Exception """ try: f = 'columnValidationLog' self.logger.log(f, "Column Length Validation Started!!") file_list = self.awsObj.listDirFiles( 'Prediction_Good_Raw_Files_Validated') for file in file_list: csv = self.awsObj.csvToDataframe( 'Prediction_Good_Raw_Files_Validated', file) if csv.shape[1] == NumberofColumns: self.awsObj.saveDataframeToCsv( 'Prediction_Good_Raw_Files_Validated', file, csv) else: self.awsObj.moveFileToFolder( 'Prediction_Good_Raw_Files_Validated', 'Prediction_Bad_Raw_Files_Validated', file) self.logger.log( f, "Invalid Column Length for the file!! File moved to Bad Raw Folder :: %s" % file) self.logger.log(f, "Column Length Validation Completed!!") except OSError: f = 'columnValidationLog' self.logger.log( f, "Error Occurred while moving the file :: %s" % OSError) raise OSError except Exception as e: f = 'columnValidationLog' self.logger.log(f, "Error Occurred:: %s" % e) raise e def deletePredictionFile(self): self.awsObj.deleteFile('Prediction_Output_File', 'Predictions.csv') def validateMissingValuesInWholeColumn(self): """ Method Name: validateMissingValuesInWholeColumn Description: This function validates if any column in the csv file has all values missing. If all the values are missing, the file is not suitable for processing. SUch files are moved to bad raw data. Output: None On Failure: Exception """ try: f = 'missingValuesInColumn' self.logger.log(f, "Missing Values Validation Started!!") file_list = self.awsObj.listDirFiles( 'Prediction_Good_Raw_Files_Validated') for file in file_list: csv = self.awsObj.csvToDataframe( 'Prediction_Good_Raw_Files_Validated', file) count = 0 for columns in csv: if (len(csv[columns]) - csv[columns].count()) == len( csv[columns]): count += 1 self.awsObj.moveFileToFolder( 'Prediction_Good_Raw_Files_Validated', 'Prediction_Bad_Raw_Files_Validated', file) self.logger.log( f, "Invalid Column Length for the file!! File moved to Bad Raw Folder :: %s" % file) break if count == 0: self.awsObj.saveDataframeToCsv( 'Prediction_Good_Raw_Files_Validated', file, csv) except OSError: f = 'missingValuesInColumn' self.logger.log( f, "Error Occurred while moving the file :: %s" % OSError) raise OSError except Exception as e: f = 'missingValuesInColumn' self.logger.log(f, "Error Occurred:: %s" % e) raise e
class File_Operation: """ This class shall be used to save the model after training and load the saved model for prediction. """ def __init__(self, file_object, logger_object): self.file_object = file_object self.logger_object = logger_object self.model_directory = 'models' self.awsObj = AwsStorageManagement() def save_model(self, model, filename): """ Method Name: save_model Description: Save the model file to directory Outcome: File gets saved On Failure: Raise Exception """ self.logger_object.log( self.file_object, 'Entered the save_model method of the File_Operation class') try: print('Start Saving Model') with io.BytesIO() as f: pickle.dump(model, f) # save the model to file f.seek(0) self.awsObj.saveObject(self.model_directory, filename + '.sav', f, 'text/html') print('Model Saved') self.logger_object.log( self.file_object, 'Model File ' + filename + ' saved. Exited the save_model method of the Model_Finder class' ) return 'success' except Exception as e: self.logger_object.log( self.file_object, 'Exception occured in save_model method of the Model_Finder class. Exception message: ' + str(e)) self.logger_object.log( self.file_object, 'Model File ' + filename + ' could not be saved. Exited the save_model method of the Model_Finder class' ) raise Exception() def load_model(self, filename): """ Method Name: load_model Description: load the model file to memory Output: The Model file loaded in memory On Failure: Raise Exception """ self.logger_object.log( self.file_object, 'Entered the load_model method of the File_Operation class') try: f = self.awsObj.loadObject(self.model_directory, filename + '.sav') print('Model load done') self.logger_object.log( self.file_object, 'Model File ' + filename + ' loaded. Exited the load_model method of the Model_Finder class' ) return f except Exception as e: self.logger_object.log( self.file_object, 'Exception occured in load_model method of the Model_Finder class. Exception message: ' + str(e)) self.logger_object.log( self.file_object, 'Model File ' + filename + ' could not be saved. Exited the load_model method of the Model_Finder class' ) raise Exception() def find_correct_model_file(self, cluster_number): """ Method Name: find_correct_model_file Description: Select the correct model based on cluster number Output: The Model file On Failure: Raise Exception """ self.logger_object.log( self.file_object, 'Entered the find_correct_model_file method of the File_Operation class' ) try: self.cluster_number = cluster_number self.folder_name = self.model_directory self.list_of_model_files = [] self.list_of_files = self.awsObj.listDirFiles(self.folder_name) for self.file in self.list_of_files: try: if (self.file.index(str(self.cluster_number)) != -1): self.model_name = self.file except: continue self.model_name = self.model_name.split('.')[0] self.logger_object.log( self.file_object, 'Exited the find_correct_model_file method of the Model_Finder class.' ) return self.model_name except Exception as e: self.logger_object.log( self.file_object, 'Exception occured in find_correct_model_file method of the Model_Finder class. Exception message: ' + str(e)) self.logger_object.log( self.file_object, 'Exited the find_correct_model_file method of the Model_Finder class with Failure' ) raise Exception()
class dBOperation: """ This class shall be used for handling all the SQL operations. """ def __init__(self): self.path = 'Training_Database' self.badFilePath = "Training_Bad_Raw_Files_Validated" self.goodFilePath = "Training_Good_Raw_Files_Validated" self.logger = App_Logger() self.awsObj = AwsStorageManagement() self.dbObj = mongoDBOperation() def createTableDb(self, DatabaseName, column_names): """ Method Name: createTableDb Description: This method creates a table in the given database which will be used to insert the Good data after raw data validation. Output: None On Failure: Raise Exception """ try: self.dbObj.createOrGetCollection(DatabaseName, 'Good_Raw_Data') file = 'DbTableCreateLog' self.logger.log(file, "Tables created successfully!!") file = 'DataBaseConnectionLog' self.logger.log(file, "Closed %s database successfully" % DatabaseName) except Exception as e: file = 'DbTableCreateLog' self.logger.log(file, "Error while creating table: %s " % e) file = 'DataBaseConnectionLog' self.logger.log(file, "Closed %s database successfully" % DatabaseName) raise e def insertIntoTableGoodData(self, Database): """ Method Name: insertIntoTableGoodData Description: This method inserts the Good data files from the Good_Raw folder into the above created table. Output: None On Failure: Raise Exception """ goodFilePath = self.goodFilePath badFilePath = self.badFilePath onlyfiles = self.awsObj.listDirFiles(goodFilePath) log_file = 'DbInsertLog' self.dbObj.dropCollection(Database, 'Good_Raw_Data') print(onlyfiles) for file in onlyfiles: try: df_csv = self.awsObj.csvToDataframe(self.goodFilePath, file) print('df_csv: ', df_csv.shape) self.dbObj.dataframeToRecords(Database, 'Good_Raw_Data', df_csv) except Exception as e: self.logger.log(log_file, "Error while creating table: %s " % e) self.awsObj.moveFileToFolder(goodFilePath, badFilePath, file) self.logger.log(log_file, "File Moved Successfully %s" % file) print('Data pushed to mongodb...') def selectingDatafromtableintocsv(self, Database): """ Method Name: selectingDatafromtableintocsv Description: This method exports the data in GoodData table as a CSV file. in a given location. above created . Output: None On Failure: Raise Exception """ self.fileFromDb = 'Training_FileFromDB' self.fileName = 'InputFile.csv' self.awsObj.createS3Directory(self.fileFromDb) log_file = 'ExportToCsv' try: tmp_csv = self.dbObj.recordsToDataFrame(Database, 'Good_Raw_Data') self.awsObj.saveDataframeToCsv('Training_FileFromDB', self.fileName, tmp_csv) self.logger.log(log_file, "File exported successfully!!!") print('Saving data to final csv') except Exception as e: self.logger.log(log_file, "File exporting failed. Error : %s" % e)
class train_validation: def __init__(self, path): self.raw_data = Raw_Data_validation(path) self.dataTransform = dataTransform() self.dBOperation = dBOperation() self.file_object = 'Training_Main_Log' self.log_writer = logger.App_Logger() self.emailObj = email() self.awsObj = AwsStorageManagement() def train_validation(self): try: self.log_writer.log(self.file_object, 'Start of Validation on files for Training!!') # extracting values from prediction schema LengthOfDateStampInFile, LengthOfTimeStampInFile, column_names, noofcolumns = self.raw_data.valuesFromSchema( ) # getting the regex defined to validate filename regex = self.raw_data.manualRegexCreation() # validating filename of prediction files self.raw_data.validationFileNameRaw(regex, LengthOfDateStampInFile, LengthOfTimeStampInFile) # validating column length in the file self.raw_data.validateColumnLength(noofcolumns) # validating if any column has all values missing self.raw_data.validateMissingValuesInWholeColumn() self.log_writer.log(self.file_object, "Raw Data Validation Complete!!") self.log_writer.log(self.file_object, "Starting Data Transforamtion!!") # below function adds quotes to the '?' values in some columns. self.dataTransform.addQuotesToStringValuesInColumn() self.log_writer.log(self.file_object, "DataTransformation Completed!!!") self.log_writer.log( self.file_object, "Creating Training_Database and tables on the basis of given schema!!!" ) # create database with given name, if present open the connection! Create table with columns given in schema self.log_writer.log(self.file_object, "Table creation Completed!!") self.log_writer.log(self.file_object, "Insertion of Data into Table started!!!!") # insert csv files in the table self.dBOperation.insertIntoTableGoodData('mushroomClassifierDB') self.log_writer.log(self.file_object, "Insertion in Table completed!!!") self.log_writer.log(self.file_object, "Deleting Good Data Folder!!!") # Delete the good data folder after loading files in table self.raw_data.deleteExistingGoodDataTrainingFolder() self.log_writer.log(self.file_object, "Good_Data folder deleted!!!") self.log_writer.log( self.file_object, "Moving bad files to Archive and deleting Bad_Data folder!!!") # Move the bad files to archive folder self.raw_data.moveBadFilesToArchiveBad() self.log_writer.log( self.file_object, "Bad files moved to archive!! Bad folder Deleted!!") self.log_writer.log(self.file_object, "Validation Operation completed!!") self.log_writer.log(self.file_object, "Extracting csv file from table") # export data in table to csvfile self.dBOperation.selectingDatafromtableintocsv( 'mushroomClassifierDB') # Triggering Email msg = MIMEMultipart() msg['Subject'] = 'MushroomTypeClassifier - Train Validation | ' + str( datetime.now()) file_list = self.awsObj.listDirFiles( 'Training_Bad_Raw_Files_Validated') if len(file_list) >= 1: file_str = ','.join(file_list) else: file_str = 'No Bad Files' body = 'Model Train Validation Done Successfully... <br><br> Fault File List: <br>' + file_str + '<br><br>Thanks and Regards, <br> Rahul Garg' msg.attach(MIMEText(body, 'html')) to_addr = ['*****@*****.**'] self.emailObj.trigger_mail(to_addr, [], msg) except Exception as e: raise e