def main():
filename = 'Test_Files/DatabaseManagerTest_15.csv'
ingestor = Ingestor(filename)
ingestor.readCSV()
print("Header of csv file")
print(ingestor.getCSVHeaders())
tempHeaders = ingestor.getCSVHeaders()
searchCritera = [tempHeaders[2], tempHeaders[3], tempHeaders[5]]
searchCritera = ingestor.getHeaderIndex(searchCritera, tempHeaders)
print("\nDictionary of search critera and their indexes in the csv")
print(searchCritera)
print("\nPrint raw list from csv")
print(ingestor.getRowAt(1))
ingestor.searchRows(searchCritera, ingestor.getRows())
print("\nPrint filtered list from unfiltered row")
print(ingestor.getRowAt(1))
print(ingestor.getRowAt(2))
print(ingestor.getRowAt(3))
print(ingestor.getRowAt(4))
print("\nNumber of columns")
print(ingestor.getNumberOfHeaders())
print("\nNumber of rows")
print(ingestor.getNumberOfRows())
print(
"\nUpdating file to a csv in project folder names newList.csv. Expected:False"
)
print(
ingestor.updateFileLoc(
"/home/anthonym/Documents/SchoolWork/SoftwareEngineering/newList.csv"
))
print("\nUpdating file to a csv with no name. Expected:false")
print(ingestor.updateFileLoc(""))
print(
"\nUpdating file to a csv with location Test_Files/DatabaseManagerTest_1000.csv. Expected:true"
)
print(ingestor.updateFileLoc("Test_Files/DatabaseManagerTest_1000.csv"))
class IngestorTest(unittest.TestCase):
def setUp(self):
filename = 'Test_Files/DatabaseManagerTest_15.csv'
self.ingestor = Ingestor(filename)
self.ingestor.readCSV()
def test_headers(self):
headerList = ["Street Address","owner's first name","last_name","Long","email","phone Number","Loan Amount","Lat"]
#self.assertEqual(self.ingestor.getCSVHeaders(),headerList)
def test_search_headers(self):
searchCritera = ["last_name","Long","phone Number"]
expectedRetun = [["last_name",2],["Long",3],["phone_Number",5]]
self.assertEqual(self.ingestor.getHeaderIndex(searchCritera,self.ingestor.getCSVHeaders()),expectedRetun)
def test_get_row(self):
expectedRetun = ["8 Hoard Court","Samuele","Gulliver","-64.1305924","*****@*****.**","+54 (656) 804-6029","$14,895.21 ","-31.4325479"]
self.assertEqual(self.ingestor.getRowAt(0),expectedRetun)
def test_get_row_filtered(self):
searchCritera = [["last_name",2],["Long",3],["phone_Number",5]]
expectedRetun = ["Gulliver","-64.1305924","+54 (656) 804-6029"]
self.ingestor.searchRows(searchCritera,self.ingestor.getRows())
self.assertEqual(self.ingestor.getRowAt(0),expectedRetun)
expectedRetun = ["Scoullar","121.5570313","+63 (634) 506-0432"]
self.assertEqual(self.ingestor.getRowAt(4),expectedRetun)
def test_number_of_headers(self):
self.assertEqual(self.ingestor.getNumberOfHeaders(),8)
def test_number_of_row(self):
self.assertEqual(self.ingestor.getNumberOfRows(),15)
def test_switch_files(self):
self.assertFalse(self.ingestor.updateFileLoc(""))
self.assertTrue(self.ingestor.updateFileLoc("Test_Files/DatabaseManagerTest_1000.csv"))
class DatabaseManagerTester(unittest.TestCase):
def setUp(self):
sqlite_file = 'test.db'
self.db = DatabaseManager(sqlite_file, '__ADMIN__')
filename = "Test_Files/DatabaseManagerTest_15.csv"
self.ingestor = Ingestor(filename)
self.ingestor.readCSV()
tempHeaders = self.ingestor.getCSVHeaders()
self.searchCritera = [
tempHeaders[0], tempHeaders[1], tempHeaders[2], tempHeaders[4],
tempHeaders[5], tempHeaders[6]
]
searchCriteraTwoD = self.ingestor.getHeaderIndex(
self.searchCritera, tempHeaders)
self.ingestor.searchRows(searchCriteraTwoD, self.ingestor.getRows())
self.searchCritera = self.db.remove_spaces(self.searchCritera)
self.new_table = 'Test_15'
def test_create_new_table(self):
self.assertTrue(
self.db.create_table_list(self.new_table, self.searchCritera,
'string'))
def test_add_row_list(self):
self.assertTrue(
self.db.add_list_of_rows(self.new_table, self.searchCritera,
self.ingestor.getRows()))
def test_get_headers(self):
expectedReturn = [
'Street_Address', "owner's_first_name", 'last_name', 'email',
'phone_Number', 'Loan_Amount'
]
self.assertEqual(self.db.get_headers(self.new_table), expectedReturn)
def test_get_table(self):
pass
def test_get_table_names(self):
tables_in_db = self.db.get_table_names()
self.assertTrue(self.new_table in tables_in_db)
def test_get_row_at_with_column(self):
column_to_use = "72 Pearson Drive"
row_from_db = self.db.get_row_at(self.new_table,
column_name=self.searchCritera[0],
column_value=column_to_use)
expectedRetun = [
u'72 Pearson Drive', u'Bartholemy', u'Parnaby',
u'*****@*****.**', u'+55 (385) 326-3642', u'$44,795.68 '
]
#The lists are the same but it doesn't think they are equal
#self.assertEqual(row_from_db,expectedRetun)
def test_get_row_at_with_rowid(self):
rowid = 3
row_from_db = self.db.get_row_at(self.new_table, row_id=rowid)
expectedRetun = [
u'72 Pearson Drive', u'Bartholemy', u'Parnaby',
u'*****@*****.**', u'+55 (385) 326-3642', u'$44,795.68 '
]
#self.assertEqual(row_from_db,expectedRetun)
def test_delete_row(self):
rowid = 9
rowToDel = self.db.get_row_at(self.new_table, row_id=rowid)
rowAfterToDel = self.db.get_row_at(self.new_table, row_id=rowid + 1)
self.db.delete_row_at(self.new_table, rowid)
self.assertEqual(self.db.get_row_at(self.new_table, row_id=rowid),
rowAfterToDel)
def test_update_row(self):
rowid = 9
old_row = self.db.get_row_at(self.new_table, row_id=rowid)
updated_row1 = [
"a house", "josh", "green", "*****@*****.**",
"228-192-2819", "$2.17"
]
self.db.update_row_at(self.new_table,
primary_key=rowid,
new_row=updated_row1)
self.assertTrue(old_row is not updated_row1)
# https://www.youtube.com/watch?v=pd-0G0MigUA
#https://stackoverflow.com/questions/17044259/python-how-to-check-if-table-exists
from DatabaseManager import DatabaseManager
from Ingestor import Ingestor
# SETTING IT EQUAL TO ':memory:' WILL HAVE IT RUN ON RAM AND NO SQLITE FILE WILL BE MADE.
sqlite_file = 'test.db'
db = DatabaseManager(sqlite_file, '__ADMIN__')
CLEAR_ON_COMPLETION = True
filename = "Test_Files/DatabaseManagerTest_15.csv"
ingestor = Ingestor(filename)
ingestor.readCSV()
tempHeaders = ingestor.getCSVHeaders()
searchCritera = [
tempHeaders[0], tempHeaders[1], tempHeaders[2], tempHeaders[4],
tempHeaders[5], tempHeaders[6]
]
searchCriteraTwoD = ingestor.getHeaderIndex(searchCritera, tempHeaders)
print("\nDictionary of search critera and their indexes in the csv")
print(searchCriteraTwoD)
ingestor.searchRows(searchCriteraTwoD, ingestor.getRows())
print("\nPrint filtered list from unfiltered row")
print(ingestor.getRowAt(0))
searchCritera = db.remove_spaces(searchCritera)
class csv_importer_popup(QtWidgets.QDialog):
#Signals when csv_importer_popup closes
importDoneSignal = QtCore.pyqtSignal('QString')
def __init__(self, window_title, db_file_loc, tables,
protected_table_prefix):
super().__init__()
#Creating the window
self.title = window_title
self.setWindowTitle(self.title)
self.protected_table_prefix = protected_table_prefix
self.tablesInDB = tables
#Database manager stuff
self.db = DatabaseManager(db_file_loc, protected_table_prefix)
#Create array with tables already in the database to be
#put in the common files radio button box
self.default_lists = []
for table in tables:
tempList = []
for columnName in self.db.get_headers(table):
tempList.append(columnName)
self.default_lists.append(tempList)
self.layout = QGridLayout()
def run_popup(self, file_loc):
#CSV file stuff
self.ingestor = Ingestor(file_loc)
self.ingestor.readCSV()
self.rows = self.ingestor.getCSVHeaders()
#Create buttons from the csv file headers that was just selected
self.generate_checkboxes(self.rows)
#Create a area that has a scroll bar
scrollArea = QScrollArea()
scrollArea.setWidget(self.csvHeaderGroup_box)
scrollArea.horizontalScrollBar().setEnabled(False)
#Create the buttons for tables that already exist in the database
self.generate_radiobuttons(self.tablesInDB)
#List of button groups
self.buttonGroups = [self.commonFileTypesGroup, self.csvHeaderGroup]
#Create label
tableNameLabel = QtWidgets.QLabel("Table Name")
#Create text field
self.tableNameField = QtWidgets.QLineEdit()
self.tableNameField.setPlaceholderText("Enter Custom Table Name")
#Create buttons
self.cancelButton = QPushButton('Cancel')
self.importButton = QPushButton('Import')
self.cancelButton.clicked.connect(self.closeWindow)
self.importButton.clicked.connect(self.importCSV)
#Create progress Bar
self.progressBar = QtWidgets.QProgressBar()
#Create the master layout which is a grid
layout = QGridLayout()
#Add widgets
#format of addWidget(widget,row,col,row span, col span)
layout.addWidget(scrollArea, 1, 1, 1, 2)
layout.addWidget(tableNameLabel, 2, 1, 1, 2)
layout.addWidget(self.tableNameField, 3, 1, 1, 2)
layout.addWidget(self.commonFileTypesGroupBox, 4, 1, 1, 2)
layout.addWidget(self.progressBar, 5, 1, 1, 2)
layout.addWidget(self.cancelButton, 6, 1)
layout.addWidget(self.importButton, 6, 2)
self.setLayout(layout)
self.resize(self.sizeHint())
def generate_checkboxes(self, button_name_list):
#Generate check_boxes
self.csvHeaderGroup = QButtonGroup()
self.csvHeaderGroup_layout = QVBoxLayout()
self.csvHeaderGroup.setExclusive(False)
self.csvHeaderGroup_box = QGroupBox('Select which headers')
self.csvHeaderGroup_layout.addStretch(1)
for button_name in button_name_list:
#Add each button to the layout from the csv file
checkbox = QCheckBox(button_name)
self.csvHeaderGroup.addButton(checkbox)
self.csvHeaderGroup_layout.addWidget(
self.csvHeaderGroup.buttons()[-1])
self.csvHeaderGroup_box.setLayout(self.csvHeaderGroup_layout)
def generate_radiobuttons(self, button_name_list):
#Generate Radio Buttons
self.commonFileTypesGroup = QButtonGroup()
self.commonFileTypesGroupLayout = QVBoxLayout()
self.commonFileTypesGroupBox = QGroupBox('Select a pre-existing table')
self.commonFileTypesGroupLayout.addStretch(1)
count = 0
for button_name in button_name_list:
#Add each button from available lists in database
radioButton = QRadioButton(button_name)
self.commonFileTypesGroup.addButton(radioButton, count)
self.commonFileTypesGroupLayout.addWidget(
self.commonFileTypesGroup.buttons()[-1])
count += 1
self.commonFileTypesGroupBox.setLayout(self.commonFileTypesGroupLayout)
def import_done(self, tableName):
#Returns what table was created
self.importDoneSignal.emit(tableName)
self.accept()
def closeWindow(self):
#Closes the window
self.reject()
def importCSV(self):
self.importButton.setEnabled(False)
self.cancelButton.setEnabled(False)
#Check if any radio buttons were pressed by checking if they were
#checked and save the number in the button group
radio_button_number = -1
special_button_number = -1
count = 0
for radioButton in self.buttonGroups[0].buttons():
if radioButton.isChecked():
radio_button_number = count
break
count += 1
for specialButton in self.buttonGroups[1].buttons(
): # Do the same for the special buttons
if specialButton.isChecked():
special_button_number = count
break
count += 1
if radio_button_number > -1:
searchCritera = self.ingestor.getHeaderIndex(
self.default_lists[radio_button_number],
self.ingestor.getCSVHeaders())
buttonText = self.buttonGroups[0].buttons(
)[radio_button_number].text()
#Check which table corresponds with the button pressed
for tableName in self.tablesInDB:
if buttonText.replace(' ', '_') == tableName:
#Uses the ingestor to search the un-filtered rows using
#this search criteria list
self.ingestor.searchRows(searchCritera,
self.ingestor.getRows())
#Check if tables exists already
if not self.db.doesTableExist(tableName):
#If not the create it with the table name
self.db.create_table_list(
tableName,
self.db.remove_spaces(
self.default_lists[radio_button_number]),
'string')
self.import_with_progress_bar(
tableName, self.ingestor.getRows(),
self.default_lists[radio_button_number])
self.import_done(tableName)
else:
try:
if self.tableNameField.text(
) == '' or self.protected_table_prefix in self.tableNameField.text(
):
raise Exception()
else:
customTableName = self.db.is_valid_string(
self.tableNameField.text().replace(' ', '_'))
if special_button_number > -1:
# Default header option not chosen, so custom lists
try:
requestedHeaders = []
for item in self.buttonGroups[1].buttons():
if item.isChecked():
requestedHeaders.append(item.text())
does_exist = self.db.doesTableExist(
customTableName)
has_same_cols = True
if not does_exist:
#If not the create it with the table name
self.db.create_table_list(
customTableName,
self.db.remove_spaces(requestedHeaders),
'string')
else:
#Tables exists. Does it have the same columns?
if not (requestedHeaders ==
self.db.get_headers(customTableName)):
has_same_cols = False
#Find the different column names
#This works by turning the lists into sets
#A set is an unordered list with no duplicate elements
#A set supports matrix operations so you can subtract the two sets
#This returns the elements that are not shared
different_cols = list(
set(
self.db.remove_spaces(
requestedHeaders)) - set(
self.db.get_headers(
customTableName)))
#Add the extra columns
for col in different_cols:
self.db.add_column(
customTableName, col, 'string')
if has_same_cols:
#New table is identical to existing one
print("same columns")
searchCritera = self.ingestor.getHeaderIndex(
requestedHeaders,
self.ingestor.getCSVHeaders())
self.ingestor.searchRows(
searchCritera, self.ingestor.getRows())
rows = self.ingestor.getRows()
self.import_with_progress_bar(
customTableName, self.ingestor.getRows(),
requestedHeaders)
else:
#New table has different columns
#Combine the headers in the lists
print("diff columns")
combinedHeaders = self.db.get_headers(
customTableName) + requestedHeaders
#Have to re order them to match the csv file
newRequestedHeaders = []
for header in self.db.remove_spaces(
self.ingestor.getCSVHeaders()):
#Find the header in the csv file
#The order matters because the primary key is needed to update the row
if header in combinedHeaders:
newRequestedHeaders.append(header)
#Get the index for the header
searchCritera = self.ingestor.getHeaderIndex(
newRequestedHeaders,
self.ingestor.getCSVHeaders())
#Filter the rows so only the requested info is there
self.ingestor.searchRows(
searchCritera, self.ingestor.getRows())
rows = self.ingestor.getRows()
#Import them nomrally
self.import_with_progress_bar(
customTableName, self.ingestor.getRows(),
newRequestedHeaders)
self.import_done(customTableName)
except Exception as er:
#General error message
print('Error message:', er.args[0])
return False
else:
raise Exception()
except:
ErrorBox = QtWidgets.QMessageBox()
choice = ErrorBox.critical(
self, 'Table Name Error',
"Table name can only have letters numbers, and underscores",
ErrorBox.Ok)
if choice == ErrorBox.Ok:
#User wants to try a new name
print("Closing")
ErrorBox.accept()
self.importButton.setEnabled(True)
self.cancelButton.setEnabled(True)
def import_with_progress_bar(self, tableName, rows_to_be_added,
column_headers):
"""
Adds the ingestor rows to the db one row at a time so the progress
bar will show the progress
"""
#Set the max value of the progress bar to the number of rows to be add
self.progressBar.setMaximum(len(rows_to_be_added))
#self.db.add_list_of_rows(tableName,self.db.remove_spaces(self.default_lists[button_number]),rows)
count = 0
for row in rows_to_be_added:
#For every row to be added add it to the db and increment the progress
#bar value by 1
count += 1
self.db.add_row_list(tableName,
self.db.remove_spaces(column_headers), row)
self.progressBar.setValue(count)
