class dataTransform:
def __init__(self):
self.goodPath = "Training_Raw_files_validated/Good_Raw"
self.logger = AppLogger()
def addQuotesToStringValuesInColumn(self):
log_file = open("Training_Logs/addQuotesToStringValuesInColumn.txt",
'a+')
try:
onlyfiles = [f for f in listdir(self.goodPath)]
for file in onlyfiles:
data = pd.read_csv(self.goodPath + "/" + file)
for column in data.columns:
count = data[column][data[column] == '?'].count()
if count != 0:
data[column] = data[column].replace('?', "'?'")
data.to_csv(self.goodPath + "/" + file,
index=None,
header=True)
self.logger.log(log_file,
" %s: Quotes added successfully!!" % file)
except Exception as e:
self.logger.log(log_file,
"Data Transformation failed because:: %s" % e)
log_file.close()
log_file.close()
class dBOperation:
def __init__(self):
self.path = 'Prediction_Database/'
self.goodFilePath = "Prediction_Raw_Files_Validated/Good_Raw"
self.badFilePath = "Prediction_Raw_Files_Validated/Bad_Raw"
self.logger = AppLogger()
def databaseConnection(self, DatabaseName):
try:
conn = sqlite3.connect(self.path + DatabaseName + '.db')
file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(file,
"Opened %s database successfully" % DatabaseName)
file.close()
except ConnectionError:
file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(
file,
"Error while connecting to database: %s" % ConnectionError)
file.close()
raise ConnectionError
return conn
def createTableDb(self, DatabaseName, column_names):
try:
conn = self.databaseConnection(DatabaseName)
conn.execute('DROP TABLE IF EXISTS GOOD_RAW_DATA')
for key in column_names.keys():
type = column_names[key]
try:
conn.execute(
'ALTER TABLE GOOD_RAW_DATA ADD COLUMN "{column_name}" {datatype}'
.format(column_name=key, datatype=type))
except:
conn.execute(
'CREATE TABLE GOOD_RAW_DATA ({column_name} {datatype})'
.format(column_name=key, datatype=type))
conn.close()
file = open("Prediction_Logs/DbTableCreateLog.txt", 'a+')
self.logger.log(file, "Tables created successfully!!")
file.close()
file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(file,
"Closed %s database successfully" % DatabaseName)
file.close()
except Exception as e:
file = open("Prediction_Logs/DbTableCreateLog.txt", 'a+')
self.logger.log(file, "Error while creating table: %s " % e)
file.close()
conn.close()
file = open("Prediction_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(file,
"Closed %s database successfully" % DatabaseName)
file.close()
raise e
def insertIntoTableGoodData(self, Database):
"""
Method Name: insertIntoTableGoodData
Description: This method inserts the Good data files from the Good_Raw folder into the
above created table.
Output: None
On Failure: Raise Exception
Written By: iNeuron Intelligence
Version: 1.0
Revisions: None
"""
conn = self.databaseConnection(Database)
goodFilePath = self.goodFilePath
badFilePath = self.badFilePath
onlyfiles = [f for f in listdir(goodFilePath)]
log_file = open("Prediction_Logs/DbInsertLog.txt", 'a+')
for file in onlyfiles:
try:
with open(goodFilePath + '/' + file, 'r') as f:
next(f)
reader = csv.reader(f, delimiter="\n")
for line in enumerate(reader):
for list_ in (line[1]):
try:
conn.execute(
'INSERT INTO Good_Raw_Data values ({values})'
.format(values=(list_)))
self.logger.log(
log_file,
" %s: File loaded successfully!!" % file)
conn.commit()
except Exception as e:
raise e
except Exception as e:
conn.rollback()
self.logger.log(log_file,
"Error while creating table: %s " % e)
shutil.move(goodFilePath + '/' + file, badFilePath)
self.logger.log(log_file, "File Moved Successfully %s" % file)
log_file.close()
conn.close()
raise e
conn.close()
log_file.close()
def selectingDatafromtableintocsv(self, Database):
self.fileFromDb = 'Prediction_FileFromDB/'
self.filename = 'InputFile.csv'
log_file = open("Prediction_Logs/ExportToCsv.txt", 'a+')
try:
conn = self.databaseConnection(Database)
sqlSelect = "SELECT * FROM GOOD_RAW_DATA"
cursor = conn.cursor()
cursor.execute(sqlSelect)
results = cursor.fetchall()
headers = [i[0] for i in cursor.description]
if not os.path.isdir(self.fileFromDb):
os.makedirs(self.fileFromDb)
csv_file = csv.writer(open(self.fileFromDb + self.filename,
'w',
newline=''),
delimiter=',',
lineterminator='\r\n',
quoting=csv.QUOTE_ALL,
escapechar='\\')
csv_file.writerow(headers)
csv_file.writerows(results)
self.logger.log(log_file, "File exported successfully!!!")
except Exception as e:
self.logger.log(log_file, "File exporting failed. Error : %s" % e)
log_file.close()
raise e
class dBOperation:
def __init__(self):
self.path = 'Training_Database/'
self.badFilePath = "Training_Raw_files_validated/Bad_Raw"
self.goodFilePath = "Training_Raw_files_validated/Good_Raw"
self.logger = AppLogger()
def databaseConnection(self, DatabaseName):
try:
conn = sqlite3.connect(self.path + DatabaseName + '.db')
file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(file,
"Opened %s database successfully" % DatabaseName)
file.close()
except ConnectionError:
file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(
file,
"Error while connecting to database: %s" % ConnectionError)
file.close()
raise ConnectionError
return conn
def createTableDb(self, DatabaseName, column_names):
try:
conn = self.databaseConnection(DatabaseName)
c = conn.cursor()
c.execute(
"SELECT COUNT(name) FROM sqlite_master WHERE type = 'table' AND name = 'GOOD_RAW_DATA'"
)
if c.fetchone()[0] == 1:
conn.close()
file = open("Training_Logs/DbTableCreateLog.txt", 'a+')
self.logger.log(file, "Tables created successfully!!")
file.close()
file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(
file, "Closed %s database successfully" % DatabaseName)
file.close()
else:
for key in column_names.keys():
type = column_names[key]
try:
conn.execute(
'ALTER TABLE GOOD_RAW_DATA ADD COLUMN "{column_name}" {datatype}'
.format(column_name=key, datatype=type))
except:
conn.execute(
'CREATE TABLE GOOD_RAW_DATA ({column_name} {datatype})'
.format(column_name=key, datatype=type))
conn.close()
file = open("Training_Logs/DbTableCreateLog.txt", 'a+')
self.logger.log(file, "Table created successfully!!")
file.close()
file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(
file, "Closed %s database successfully" % DatabaseName)
file.close()
except Exception as e:
file = open("Training_Logs/DbTableCreateLog.txt", 'a+')
self.logger.log(file, "Error while creating table: %s " % e)
file.close()
conn.close()
file = open("Training_Logs/DataBaseConnectionLog.txt", 'a+')
self.logger.log(file,
"Closed %s database successfully" % DatabaseName)
file.close()
raise e
def insertIntoTableGoodData(self, Database):
conn = self.databaseConnection(Database)
goodFilePath = self.goodFilePath
badFilePath = self.badFilePath
onlyfiles = [f for f in listdir(goodFilePath)]
log_file = open("Training_Logs/DbInsertLog.txt", 'a+')
for file in onlyfiles:
try:
with open(self.goodFilePath + '/' + file, 'r') as f:
next(f)
reader = csv.reader(f, delimiter="\n")
for line in enumerate(reader):
for list_ in (line[1]):
try:
conn.execute(
'INSERT INTO GOOD_RAW_DATA values({values})'
.format(values=list_))
self.logger.log(
log_file,
" %s: File loaded successfully!!" % file)
conn.commit()
except Exception as e:
raise e
except Exception as e:
conn.rollback()
self.logger.log(log_file,
"Error while creating table: %s " % e)
shutil.move(goodFilePath + '/' + file, badFilePath)
self.logger.log(log_file, "File Moved Successfully %s" % file)
log_file.close()
conn.close()
conn.close()
log_file.close()
def selectingDatafromtableintocsv(self, Database):
self.fileFromDb = 'Training_FileFromDB/'
self.filename = 'InputFile.csv'
log_file = open("Training_Logs/ExportToCsv.txt", 'a+')
try:
conn = self.databaseConnection(Database)
sqlSelect = 'SELECT * FROM GOOD_RAW_DATA'
cursor = conn.cursor()
cursor.execute(sqlSelect)
results = cursor.fetchall()
headers = [i[0] for i in cursor.description]
if not os.path.isdir(self.fileFromDb):
os.makedirs(self.fileFromDb)
csvFile = csv.writer(open(self.fileFromDb + self.filename,
'w',
newline=''),
delimiter=',',
lineterminator='\r\n',
quoting=csv.QUOTE_ALL,
escapechar='\\')
csvFile.writerow(headers)
csvFile.writerows(results)
self.logger.log(log_file, "File exported successfully!!!")
log_file.close()
except Exception as e:
self.logger.log(log_file, "File exporting failed. Error : %s" % e)
log_file.close()
class Prediction_Data_Validation:
def __init__(self, path):
self.Batch_Directory = path
self.schema_path = 'schema_prediction.json'
self.logger = AppLogger()
def valuesFromSchema(self):
try:
with open(self.schema_path, 'r') as f:
dic = json.load(f)
f.close()
pattern = dic['SampleFileName']
LengthOfDateStampInFile = dic['LengthOfDateStampInFile']
LengthOfTimeStampInFile = dic['LengthOfTimeStampInFile']
column_names = dic['ColName']
NumberofColumns = dic['NumberofColumns']
file = open("Prediction_Logs/valuesfromSchemaValidationLog.txt",
'a+')
message = "LengthOfDateStampInFile:: %s" % LengthOfDateStampInFile + "\t" + "LengthOfTimeStampInFile:: %s" % LengthOfTimeStampInFile + "\t " + "NumberofColumns:: %s" % NumberofColumns + "\n"
self.logger.log(file, message)
file.close()
except ValueError:
file = open("Prediction_Logs/valuesfromSchemaValidationLog.txt",
'a+')
self.logger.log(
file, "ValueError:Value not found inside schema_training.json")
file.close()
raise ValueError
except KeyError:
file = open("Prediction_Logs/valuesfromSchemaValidationLog.txt",
'a+')
self.logger.log(file,
"KeyError:Key value error incorrect key passed")
file.close()
raise KeyError
except Exception as e:
file = open("Prediction_Logs/valuesfromSchemaValidationLog.txt",
'a+')
self.logger.log(file, str(e))
file.close()
raise e
return LengthOfDateStampInFile, LengthOfTimeStampInFile, column_names, NumberofColumns
def manualRegexCreation(self):
regex = "['phising']+['\_'']+[\d_]+[\d]+\.csv"
return regex
def createDirectoryForGoodBadRawData(self):
try:
path = os.path.join("Prediction_Raw_Files_Validated/", "Good_Raw/")
if not os.path.isdir(path):
os.makedirs(path)
path = os.path.join("Prediction_Raw_Files_Validated/", "Bad_Raw/")
if not os.path.isdir(path):
os.makedirs(path)
except OSError as ex:
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Error while creating Directory %s:" % ex)
file.close()
raise OSError
def deleteExistingGoodDataTrainingFolder(self):
try:
path = 'Prediction_Raw_Files_Validated/'
if os.path.isdir(path + 'Good_Raw/'):
shutil.rmtree(path + 'Good_Raw/')
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file,
"GoodRaw directory deleted successfully!!!")
file.close()
except OSError as s:
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Error while Deleting Directory : %s" % s)
file.close()
raise OSError
def deleteExistingBadDataTrainingFolder(self):
try:
path = 'Prediction_Raw_Files_Validated/'
if os.path.isdir(path + 'Bad_Raw/'):
shutil.rmtree(path + 'Bad_Raw/')
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file,
"BadRaw directory deleted successfully!!!")
file.close()
except OSError as s:
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Error while Deleting Directory : %s" % s)
file.close()
raise OSError
def moveBadFilesToArchiveBad(self):
"""
Method Name: moveBadFilesToArchiveBad
Description: This method deletes the directory made to store the Bad Data
after moving the data in an archive folder. We archive the bad
files to send them back to the client for invalid data issue.
Output: None
On Failure: OSError
Written By: iNeuron Intelligence
Version: 1.0
Revisions: None
"""
now = datetime.now()
date = now.date()
time = now.strftime("%H%M%S")
try:
path = "PredictionArchivedBadData"
if not os.path.isdir(path):
os.makedirs(path)
source = 'Prediction_Raw_Files_Validated/Bad_Raw/'
dest = 'PredictionArchivedBadData/BadData_' + str(
date) + "_" + str(time)
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(source)
for f in files:
if f not in os.listdir(dest):
shutil.move(source + f, dest)
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Bad files moved to archive")
path = 'Prediction_Raw_Files_Validated/'
if os.path.isdir(path + 'Bad_Raw/'):
shutil.rmtree(path + 'Bad_Raw/')
self.logger.log(file, "Bad Raw Data Folder Deleted successfully!!")
file.close()
except OSError as e:
file = open("Prediction_Logs/GeneralLog.txt", 'a+')
self.logger.log(file,
"Error while moving bad files to archive:: %s" % e)
file.close()
raise OSError
def validationFileNameRaw(self, regex, LengthOfDateStampInFile,
LengthOfTimeStampInFile):
"""
Method Name: validationFileNameRaw
Description: This function validates the name of the prediction csv file as per given name in the schema!
Regex pattern is used to do the validation.If name format do not match the file is moved
to Bad Raw Data folder else in Good raw data.
Output: None
On Failure: Exception
Written By: iNeuron Intelligence
Version: 1.0
Revisions: None
"""
# delete the directories for good and bad data in case last run was unsuccessful and folders were not deleted.
self.deleteExistingBadDataTrainingFolder()
self.deleteExistingGoodDataTrainingFolder()
self.createDirectoryForGoodBadRawData()
onlyfiles = [f for f in listdir(self.Batch_Directory)]
try:
f = open("Prediction_Logs/nameValidationLog.txt", 'a+')
for filename in onlyfiles:
if (re.match(regex, filename)):
splitAtDot = re.split('.csv', filename)
splitAtDot = (re.split('_', splitAtDot[0]))
if len(splitAtDot[1]) == LengthOfDateStampInFile:
if len(splitAtDot[2]) == LengthOfTimeStampInFile:
shutil.copy(
"Prediction_Batch_files/" + filename,
"Prediction_Raw_Files_Validated/Good_Raw")
self.logger.log(
f,
"Valid File name!! File moved to GoodRaw Folder :: %s"
% filename)
else:
shutil.copy(
"Prediction_Batch_files/" + filename,
"Prediction_Raw_Files_Validated/Bad_Raw")
self.logger.log(
f,
"Invalid File Name!! File moved to Bad Raw Folder :: %s"
% filename)
else:
shutil.copy("Prediction_Batch_files/" + filename,
"Prediction_Raw_Files_Validated/Bad_Raw")
self.logger.log(
f,
"Invalid File Name!! File moved to Bad Raw Folder :: %s"
% filename)
else:
shutil.copy("Prediction_Batch_files/" + filename,
"Prediction_Raw_Files_Validated/Bad_Raw")
self.logger.log(
f,
"Invalid File Name!! File moved to Bad Raw Folder :: %s"
% filename)
f.close()
except Exception as e:
f = open("Prediction_Logs/nameValidationLog.txt", 'a+')
self.logger.log(f,
"Error occured while validating FileName %s" % e)
f.close()
raise e
def validateColumnLength(self, NumberofColumns):
"""
Method Name: validateColumnLength
Description: This function validates the number of columns in the csv files.
It is should be same as given in the schema file.
If not same file is not suitable for processing and thus is moved to Bad Raw Data folder.
If the column number matches, file is kept in Good Raw Data for processing.
The csv file is missing the first column name, this function changes the missing name to "Wafer".
Output: None
On Failure: Exception
Written By: iNeuron Intelligence
Version: 1.0
Revisions: None
"""
try:
f = open("Prediction_Logs/columnValidationLog.txt", 'a+')
self.logger.log(f, "Column Length Validation Started!!")
for file in listdir('Prediction_Raw_Files_Validated/Good_Raw/'):
csv = pd.read_csv("Prediction_Raw_Files_Validated/Good_Raw/" +
file)
if csv.shape[1] == NumberofColumns:
#csv.rename(columns={"Unnamed: 0": "Wafer"}, inplace=True)
csv.to_csv("Prediction_Raw_Files_Validated/Good_Raw/" +
file,
index=None,
header=True)
else:
shutil.move(
"Prediction_Raw_Files_Validated/Good_Raw/" + file,
"Prediction_Raw_Files_Validated/Bad_Raw")
self.logger.log(
f,
"Invalid Column Length for the file!! File moved to Bad Raw Folder :: %s"
% file)
self.logger.log(f, "Column Length Validation Completed!!")
except OSError:
f = open("Prediction_Logs/columnValidationLog.txt", 'a+')
self.logger.log(
f, "Error Occured while moving the file :: %s" % OSError)
f.close()
raise OSError
except Exception as e:
f = open("Prediction_Logs/columnValidationLog.txt", 'a+')
self.logger.log(f, "Error Occured:: %s" % e)
f.close()
raise e
f.close()
def validateMissingValuesInWholeColumn(self):
try:
f = open("Prediction_Logs/missingValuesInColumn.txt", 'a+')
self.logger.log(f, "Missing Values Validation Started!!")
for file in listdir('Prediction_Raw_Files_Validated/Good_Raw/'):
csv = pd.read_csv('Prediction_Raw_Files_Validated/Good_Raw/' +
file)
count = 0
for columns in csv:
if (len(csv[columns]) - csv[columns].count()) == len(
csv[columns]):
count += 1
shutil.move(
"Prediction_Raw_Files_Validated/Good_Raw/" + file,
"Prediction_Raw_Files_Validated/Bad_Raw")
self.logger.log(
f,
"Invalid Column Length for the file!! File moved to Bad Raw Folder :: %s"
% file)
break
if count == 0:
csv.to_csv("Prediction_Raw_Files_Validated/Good_Raw/" +
file,
index=None,
header=True)
except OSError:
f = open("Prediction_Logs/missingValuesInColumn.txt", 'a+')
self.logger.log(
f, "Error Occured while moving the file :: %s" % OSError)
f.close()
raise OSError
except Exception as e:
f = open("Prediction_Logs/missingValuesInColumn.txt", 'a+')
self.logger.log(f, "Error Occured:: %s" % e)
f.close()
raise e
f.close()
def deletePredictionFile(self):
if os.path.exists('Prediction_Output_File/Predictions.csv'):
os.remove('Prediction_Output_File/Predictions.csv')
class Raw_Data_Validation:
def __init__(self, path):
self.Batch_Directory = path
self.logger = AppLogger()
self.schema_path = 'schema_training.json'
def valueFromSchema(self):
try:
with open(self.schema_path, 'r') as f:
dic = json.load(f)
f.close()
pattern = dic['SampleFileName']
LengthOfDateStampInFile = dic['LengthOfDateStampInFile']
LengthOfTimeStampInFile = dic['LengthOfTimeStampInFile']
NumberofColumns = dic['NumberofColumns']
column_names = dic['ColName']
file = open("Training_Logs/valuesfromSchemaValidationLog.txt",
'a+')
message = "LengthOfDateStampInFile:: %s" % LengthOfDateStampInFile + "\t" + "LengthOfTimeStampInFile:: %s" % LengthOfTimeStampInFile + "\t " + "NumberofColumns:: %s" % NumberofColumns + "\n"
self.logger.log(file, message)
file.close()
except ValueError:
file = open("Training_Logs/valuesfromSchemaValidationLog.txt",
'a+')
self.logger.log(
file, "ValueError:Value not found inside schema_training.json")
file.close()
raise ValueError
except KeyError:
file = open("Training_Logs/valuesfromSchemaValidationLog.txt",
'a+')
self.logger.log(file,
"KeyError:Key value error incorrect key passed")
file.close()
raise KeyError
except Exception as e:
file = open("Training_Logs/valuesfromSchemaValidationLog.txt",
'a+')
self.logger.log(file, str(e))
file.close()
raise e
return LengthOfDateStampInFile, LengthOfTimeStampInFile, column_names, NumberofColumns
def manualRegexCreation(self):
regex = "['phising']+['\_'']+[\d_]+[\d]+\.csv"
return regex
def createDirectoryForGoodBadRawData(self):
try:
path = os.path.join("Training_Raw_files_validated/", "Good_Raw/")
if not os.path.isdir(path):
os.makedirs(path)
path = os.path.join("Training_Raw_files_validated/", "Good_Raw/")
if not os.path.isdir(path):
os.makedirs(path)
except OSError as ex:
file = open("Training_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Error while creating directory %s" % ex)
file.close()
raise OSError
def deleteExistingGoodDataTrainingFolder(self):
try:
path = 'Training_Raw_files_validated/'
if os.path.isdir(path + 'Good_Raw/'):
shutil.rmtree(path + 'Good_Raw/')
file = open("Training_Logs/GeneralLog.txt", 'a+')
self.logger.log(file,
"GoodRaw directory deleted successfully!!!")
file.close()
except OSError as s:
file = open("Training_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Error while Deleting Directory : %s" % s)
file.close()
raise OSError
def deleteExistingBadDataTrainingFolder(self):
try:
path = 'Training_Raw_files_validated/'
if os.path.isdir(path + 'Bad_raw/'):
shutil.rmtree(path + 'Bad_raw/')
file = open("Training_Logs/GeneralLog.txt", 'a+')
self.logger.log(file,
"BadRaw directory deleted successfully!!!")
file.close()
except OSError as s:
file = open("Training_Logs/GeneralLog.txt", 'a+')
self.logger.log(file, "Error while Deleting Directory : %s" % s)
file.close()
raise OSError
def moveBadFilesToArchiveBad(self):
now = datetime.now()
date = now.date()
time = now.strftime("%H%M%S")
try:
source = 'Training_Raw_files_validated/Bad_Raw/'
if os.path.isdir(source):
path = "TrainingArchiveBadData"
if not os.path.isdir(path):
os.makedirs(path)
dest = 'TrainingArchiveBadData/BadData_' + str(
date) + "_" + str(time)
if not os.path.isdir(dest):
os.makedirs(dest)
files = os.listdir(source)
for f in files:
if f not in os.listdir(dest):
shutil.move(source + f, dest)
file = "Training_Logs/GeneralLog.txt", 'a+'
self.logger.log(file, "Bad files moved to archive")
path = 'Training_Raw_files_validated/'
if os.path.isdir(path + 'Bad_Raw/'):
shutil.rmtree(path + 'Bad_Raw/')
self.logger.log(file,
"Bad Raw Data Folder Deleted successfully!!")
file.close()
except Exception as e:
file = open("Training_Logs/GeneralLog.txt", 'a+')
self.logger.log(file,
"Error while moving bad files to archive:: %s" % e)
file.close()
raise e
def validationFileNameRaw(self, regex, LengthOfDateStampInFile,
LengthOfTimeStampInFile):
self.deleteExistingBadDataTrainingFolder()
self.deleteExistingGoodDataTrainingFolder()
onlyfiles = [f for f in listdir(self.Batch_Directory)]
try:
self.createDirectoryForGoodBadRawData()
f = open("Training_Logs/nameValidationLog.txt", 'a+')
for filename in onlyfiles:
if (re.match(regex, filename)):
splitAtDot = re.split('.csv', filename)
splitAtDot = (re.split('_', splitAtDot[0]))
if len(splitAtDot[1]) == LengthOfDateStampInFile:
if len(splitAtDot[2]) == LengthOfTimeStampInFile:
shutil.copy(
"Training_Batch_Files/" + filename,
"Training_Raw_files_validated/Good_Raw")
self.logger.log(
f,
"Valid File name!! File moved to GoodRaw Folder :: %s"
% filename)
else:
shutil.copy(
"Training_Batch_Files/" + filename,
"Training_Raw_files_validated/Bad_Raw")
self.logger.log(
f,
"InValid File name!! File moved to BadRaw Folder :: %s"
% filename)
else:
shutil.copy("Training_Batch_Files/" + filename,
"Training_Raw_files_validated/Bad_Raw")
self.logger.log(
f,
"InValid File name!! File moved to BadRaw Folder :: %s"
% filename)
else:
shutil.copy("Training_Batch_Files/" + filename,
"Training_Raw_files_validated/Bad_Raw")
self.logger.log(
f,
"InValid File name!! File moved to BadRaw Folder :: %s"
% filename)
f.close()
except Exception as e:
f = open("Training_Logs/nameValidationLog.txt", 'a+')
self.logger.log(f,
"Error occured while validating FileName %s" % e)
f.close()
raise e
def validateColumnLength(self, NumberofColumns):
try:
f = open("Training_Logs/columnValidationLog.txt", 'a+')
self.logger.log(f, "Column length Validation Started!!!")
for file in listdir('Training_Raw_files_validated/Good_Raw/'):
csv = pd.read_csv('Training_Raw_files_validated/Good_Raw/' +
file)
if csv.shape[1] == NumberofColumns:
pass
else:
shutil.move(
"Training_Raw_files_validated/Good_Raw/" + file,
"Training_Raw_files_validated/Bad_Raw")
self.logger.log(
f,
"Invalid Column Length for the file!! File moved to Bad Raw Folder :: %s"
% file)
self.logger.log(f, "Column Length Validation Completed!!")
except OSError:
f = open("Training_Logs/columnValidationLog.txt", 'a+')
self.logger.log(
f, "Error Occured while moving the file :: %s" % OSError)
f.close()
raise OSError
except Exception as e:
f = open("Training_Logs/columnValidationLog.txt", 'a+')
self.logger.log(f, "Error Occured %s" % e)
f.close()
raise e
f.close()
def validateMissingValuesInWholeColumn(self):
try:
f = open("Training_Logs/missingValuesInColumn.txt", 'a+')
self.logger.log(f, "Missing Values Validation Started!!")
for file in listdir('Training_Raw_files_validated/Good_Raw/'):
csv = pd.read_csv('Training_Raw_files_validated/Good_Raw/' +
file)
count = 0
for columns in csv:
if (len(csv[columns]) - csv[columns].count()) == len(
csv[columns]):
count += 1
shutil.move(
"Training_Raw_files_validated/Good_Raw/" + file,
"Training_Raw_files_validated/Bad_Raw")
break
if count == 0:
csv.to_csv("Training_Raw_files_validated/Good_Raw/" + file,
index=None,
header=True)
except OSError:
f = open("Training_Logs/missingValuesInColumn.txt", 'a+')
self.logger.log(
f, "Error Occured while moving the file :: %s" % OSError)
f.close()
raise OSError
except Exception as e:
f = open("Training_Logs/missingValuesInColumn.txt", 'a+')
self.logger.log(f, "Error Occured:: %s" % e)
f.close()
raise e