class DataDict_Model():
class_alias = "datadict" #Mostly for logging and error handling
debug = True
error_message = None
continue_after_error = None #By default, errors will cause the module to abort and unload. Set continue_after_error to True to override this default.
abort_all = None #By default, an error will kill the currently running module, but will not bring down the whole application (user can still continue working with the Tkinter menu system)
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
datadict_file_to_load_from = None #Data dictionary file name and path, which will be loaded into the entry grid if user requests.
#datadict_loadfile_displaybox = None #A textbox where the data dict file to be loaded will be displayed
datadict_file_to_save_to = None #Data dictionary file name and path, to which the metadata will be saved if user requests.
#datadict_savefile_displaybox = None #A textbox where the save-to data dict file will be displayed
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
title = '' #Title to be displayed in the Frame object
mem_or_rec = None #Is this datadict associated with a Record File or a Memory File?
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
#Arrays to hold the contents of the Data Dictionary:
meta_columns = None #List of column names within the data dictionary CSV (column_name, column_width, etc.)
meta_rownums = None #For convenience, a simple List of integers that will be displayed vertically on the left side of the Data Dict grid display.
meta_values = None #List of values for each cell within the data dictionary CSV (actually a list of lists - a list of "rows" from the data dictionary CSV)
meta_values_after_edit = [] #List of values from the Entry Grid, AFTER the user has modified the values and clicked SAVE.
btnLoadDictFile = None
btnSaveToDictFile = None
def __init__(self, parent_window, controller, datadict_file_to_load_from=None, show_view=None, title='Data dictionary', **kw): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
#Frame.__init__(self, parent_window)
self.parent_window = parent_window #parent_window is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
print("\nIn DataDict._init_, datadict_file_to_load_from = '" + str(self.datadict_file_to_load_from) + "' -- and show_view=" + str(self.show_view))
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\nIn DataDict._init_: datadict_file_to_load_from = '" + str(self.datadict_file_to_load_from) + "' -- and show_view=" + str(self.show_view), True, True, True )
self.title = title
self.show_view = show_view
self.datadict_file_to_load_from = datadict_file_to_load_from
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor=gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width=gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height=gl_frame_height
if self.check_key_exists("mem_or_rec", **kw):
self.mem_or_rec = str(kw["mem_or_rec"]).lower().strip()
#self.test_error_handler()
def read_datadict_file_into_arrays(self, file_name):
'''A data dictionary CSV file is read into arrays (which are properties of this Model object). From the arrays they can be written back to a CSV file later. '''
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
self.logobject.logit("\n" + "In DataDict.read_datadict_file_into_arrays, datadict_file_to_load_from = '" + str(file_name) + "'", True, True, True)
#print("\n" + "In DataDict.read_datadict_file_into_arrays, datadict_file_to_load_from = '" + str(file_name) + "'", True, True)
self.meta_columns = []
self.meta_rownums = []
self.meta_values = []
datadict_arrays = {}
try:
f = open(file_name, "r")
content = f.readlines()
i = 0
for line in content:
print(line)
if(i==0): #Header row in the CSV file
self.meta_columns = line.split(",")
else:
meta_temp = line.split(",") #meta_temp is a LIST consisting of one row from the data dictionary
self.meta_values.append(meta_temp) #meta_values is a LIST of LISTS, consisting of one "outer" list representing all the rows from the data dictionary, and an "inner" list consisting of the cell values for a single row.
self.meta_rownums.append(str(i))
i += 1
except:
self.error_message = "Failed to read data dictionary into memory"
exc_type, exc_value, exc_traceback = sys.exc_info() #sys.exc_inf() returns a 3-item tuple with exception information
self.controller.handle_error(self.error_message, exc_type, exc_value, exc_traceback, self.continue_after_error, self.abort_all, self.class_alias) #Allow the Bigmatch Controller (main.py) to deal with the error, based on continue_after_error and abort_all.
finally:
pass
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
def write_datadict_from_entrygrid(self, file_name=None):
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
if file_name is None:
file_name = self.datadict_file_to_save_to
print("\n Top of write_datadict_from_entrygrid(), file_name is '%s'" % (file_name) )
#Call the View object to retrieve Entry Grid values (after they have been update by the user).
try:
if file_name:
del self.meta_values_after_edit[:]
self.meta_values_after_edit = [] #Clear out any previous entries
#Load the entry grid items into the meta_values_after_edit list:
self.meta_values_after_edit = self.view_object.retrieve_grid_values()
if self.meta_values_after_edit:
newrow = []
if self.debug: self.logobject.logit("Dict file in write_datadict_from_entrygrid(): %s ...File will be deleted if it already exists." % ( str(file_name) ), True, True, True )
#print("Dict file in write_datadict_from_entrygrid(): %s ...File will be deleted if it already exists." % ( str(file_name) ))
if os.path.isfile(file_name):
os.remove(file_name) #Should not need to delete the file, but it has been appending new text at end of file instead of erasing existing text and re-writing-- even though the file open uses "w" not "w+", and it is closed after every write session.
with open(file_name, 'w', encoding='UTF8') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator='\n') #os.linesep) #chr(13) + chr(10)
#Get a list of column names retrieved from the Entry Grid -- these are CSV header columns.
j = 0;
for col in self.view_object.entry_grid.col_list:
newcol = str(col).replace("\n", "").strip()
newcol = newcol.replace(chr(10), "")
newcol = newcol.replace(chr(13), "")
if (j == 0 and newcol == ''): #First "column name" is usually blank because it is the row-numbers column, which has no header caption
pass
else:
newrow.append(newcol)
j += 1
csvwriter.writerow(newrow) #Write the Column Headers to the CSV file
newrow = []
i = 0
for row in self.meta_values_after_edit:
if self.debug:
self.logobject.logit("\n In write_datadict(), ROW: %s (length is %s and type is %s" % (str(i), str(len(row)), str(type(row)) ), True, True, True)
self.logobject.logit(row, True, True, True)
rowcheck = ', '.join(row)
rowcheck = rowcheck.replace(",", "").replace("'", "").strip()
if self.debug: self.logobject.logit("Rowcheck: %s" % (rowcheck), True, True, True)
if rowcheck == "":
if self.debug: self.logobject.logit("ROW IS EMPTY. SKIP IT.", True, True, True)
continue #If row is blank, skip it!
else:
pass #if self.debug: self.logobject.logit("Rowcheck populated: %s" % (rowcheck), True, True, True )
newrow = []
j = 0
for col in row:
#if col.find("\n") != -1:
# print("Reassigning col %s in row %s to a string value," % (str(j), str(i)) )
# #row[i] = newcol #str(col)
#else:
newcol = str(col).replace("\n", "")
newcol = newcol.replace(chr(10), "")
newcol = newcol.replace(chr(13), "")
#print(newcol)
newrow.append(newcol)
j += 1
#print(newrow)
if len(newrow) == 1:
if str(row[0]) == "\n": #blank row has just newline feed
row[0] = row[0].replace("\n", "")
if not row or len(row) == 0:
continue
#Write this row to the CSV file:
csvwriter.writerow(newrow)
i += 1
csvfile.close()
except:
self.error_message = "Failed to write data dictionary info to disk file"
exc_type, exc_value, exc_traceback = sys.exc_info() #sys.exc_inf() returns a 3-item tuple with exception information
self.controller.handle_error(self.error_message, exc_type, exc_value, exc_traceback, self.continue_after_error, self.abort_all, self.class_alias) #Allow the Bigmatch Controller (main.py) to deal with the error, based on continue_after_error and abort_all.
finally:
pass
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
def load_standard_datadict_headings(self):
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
self.meta_columns = []
print("\nIn load_standard_datadict_headings(), loading standard datadict headings")
try:
ddcommon = Datadict_Common()
self.meta_columns = ddcommon.load_standard_datadict_headings(["bigmatch"]) #Pass a LIST object of context keywords to configure the data dictionary for the current context
print(self.meta_columns)
except:
#pass
self.meta_columns = ['column_name', 'start_pos', 'width', 'unique_id_yn', 'matchfield_yn', 'bigmatch_type', 'data_format', 'comments']
print(self.meta_columns)
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
return self.meta_columns
def instantiate_view_object(self, container):
self.view_object = DataDict_View(container, self)
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
return self.view_object
def display_view(self, container=None):
#Most often, we will display the Data Dictionary form in the main class's BigCanvas.BigFrame "container".
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
print("In DataDict.Display_View, Type of Container is '%s'" % (str(type(container))) )
try:
kw_dict = {"width":self.frame_width, "background":self.bgcolor, "borderwidth":2, "padx":3, "pady":3}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
#self.controller.clear_main_canvas()
if self.datadict_file_to_load_from:
#Function "read_datadict_file_into_arrays" extracts the contents of the Data Dictionary file and stores it in arrays. The arrays are them passed to TkEntry to be displayed.
print("\n" + "In DataDict.display_view(), about to run read_datadict_file_into_arrays() with '" + str(self.datadict_file_to_load_from) + "'" )
self.read_datadict_file_into_arrays(self.datadict_file_to_load_from)
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(container) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container, self.datadict_file_to_load_from)
self.display_user_buttons(container)
print("\n In DataDict.display_view(), about to call view_object.initUI().")
#Display the VIEW for this data dictionary
self.view_object.initUI(**kw_dict) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
self.view_object.display_datadict_in_grid(**kw_dict) #When first initiated, the View object will call this during its InitUI() method. But to refresh it afterwards, call display_datadict_in_grid() explicitly.
self.controller.refresh_main_canvas()
except:
raise
self.error_message = "Error displaying data dictionary"
exc_type, exc_value, exc_traceback = sys.exc_info() #sys.exc_inf() returns a 3-item tuple with exception information
self.controller.handle_error(self.error_message, exc_type, exc_value, exc_traceback, self.continue_after_error, self.abort_all, self.class_alias) #Allow the Bigmatch Controller (main.py) to deal with the error, based on continue_after_error and abort_all.
finally:
pass
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
try:
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
self.button_frame = Frame(container)
if str(type(container)).lower().find(".tk") == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(self.button_frame, "DataDict_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(background=self.bgcolor)
#Note: the specified CSV data dictionary file is loaded automatically when the user selects it (because update_filepath() is called )
#self.btnLoadDictFile = Button(self.button_frame, text="Load Data Dictionary from CSV File", width=30, command=self.display_view)
#self.btnLoadDictFile.grid(row=0, column=0, sticky=W)
#self.btnLoadDictFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to load
self.btnSaveToDictFile = Button(self.button_frame, text="Save Data Dictionary to CSV File", width=30, command=self.write_datadict_from_entrygrid)
self.btnSaveToDictFile.grid(row=0, column=1, sticky=W)
self.btnSaveToDictFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to save as
#Create a Message Region where we can display text temporarily, such as "File was successfully saved"
msg = ""
self.message_region = Message(self.button_frame, text=msg) #self.message_region = Label(self.button_frame, text=msg)
self.message_region.grid(row=0, column=5, sticky=E)
kw = {"anchor":E, "width":800, "foreground":"dark green", "background":self.bgcolor, "borderwidth":1, "font":("Arial", 11, "bold"), "padx":8, "pady":3 }
self.message_region.configure(**kw)
except:
self.error_message = "Error displaying data dictionary"
exc_type, exc_value, exc_traceback = sys.exc_info() #sys.exc_inf() returns a 3-item tuple with exception information
self.controller.handle_error(self.error_message, exc_type, exc_value, exc_traceback, self.continue_after_error, self.abort_all, self.class_alias) #Allow the Bigmatch Controller (main.py) to deal with the error, based on continue_after_error and abort_all.
finally:
pass
if self.check_for_fatal_error(): return None #Do not return anything from this function if error status is severe
def update_message_region(self, text='', clear_after_ms=5000, **kw):
self.message_region.configure(text=text)
self.message_region.after(clear_after_ms, self.clear_message_region)
def clear_message_region(self):
self.message_region.configure(text="")
def display_openfile_dialogs(self, container, default_filepath=''):
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
try:
file_types = [('All files', '*'), ('Text files', '*.csv;*.txt')]
kw_fpath = {"bgcolor":self.bgcolor, "frame_width":"", "frame_height":"", "file_category":"datadict"}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(self.parent_window, self, self.controller, "Data dictionary file to load:", open_or_save_as, "DataDictToLoad", file_types, **kw_fpath)
self.filepathobj_load_from.display_view(container) #Display the dialog for user to select a data dict file
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(self.parent_window, self, self.controller, "Save data dictionary to file:", open_or_save_as, "DataDictToSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(container) #Display the dialog for user to Ave As... a new data dict file
except:
self.error_message = "Error displaying data dictionary"
exc_type, exc_value, exc_traceback = sys.exc_info() #sys.exc_inf() returns a 3-item tuple with exception information
self.controller.handle_error(self.error_message, exc_type, exc_value, exc_traceback, self.continue_after_error, self.abort_all, self.class_alias) #Allow the Bigmatch Controller (main.py) to deal with the error, based on continue_after_error and abort_all.
finally:
pass
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
def update_filepath(self, file_name_with_path='', callback_string='', alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
self.logobject.logit("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string), True, True )
#print("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string))
if callback_string.lower().strip()[:4] == "load" or callback_string.lower().strip()[:4] == "open":
self.datadict_file_to_load_from = file_name_with_path
#if file_name_with_path: #Commented this out because btnLoadDictFile seems to be unnecessary - we automatically load the Data Dict
# self.btnLoadDictFile.config(state=NORMAL)
#Refresh the Data Dictionary view when the user selects a new DataDict file.
if self.datadict_file_to_load_from:
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid() #Remove all existing values from the grid
elif callback_string.lower().strip()[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.datadict_file_to_save_to = file_name_with_path
if self.datadict_file_to_save_to:
self.btnSaveToDictFile.config(state=NORMAL)
else:
self.btnSaveToDictFile.config(state=DISABLED)
#Display a temporary message notifying the user that their file was created.
self.update_message_region("Click 'Save Data Dictionary to CSV File' to save changes to the selected file")
self.update_master_paths(file_name_with_path)
self.update_initial_dir_for_file_open_dialogs()
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
def update_master_paths(self, file_name_with_path):
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
if self.mem_or_rec == "mem":
self.controller.mem_datadict_last_opened = file_name_with_path #The controller tracks last file opened of this type, so that when the user is again prompted to open the same type of file, we can default this value in.
if self.mem_or_rec == "rec":
self.controller.rec_datadict_last_opened = file_name_with_path #The controller tracks last file opened of this type, so that when the user is again prompted to open the same type of file, we can default this value in.
print("\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s" % (self.controller.dir_last_opened, self.controller.datadict_dir_last_opened, self.controller.rec_datadict_last_opened, self.controller.mem_datadict_last_opened) )
if self.check_for_fatal_error(): return None #Do not execute this function if error status is severe
def update_initial_dir_for_file_open_dialogs(self):
'''In addition to tracking "last file opened" at the main controller level, we also want to notify every FilePath object when the user has opened a new file, so that they can adjust thir Initial DIr properties to the location just opened.'''
self.filepathobj_load_from.calc_initial_dir_for_file_open(self.datadict_file_to_load_from, "datadict", "mem")
self.filepathobj_save_to.calc_initial_dir_for_file_open(self.datadict_file_to_save_to, "datadict", "rec")
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
def check_for_fatal_error(self):
return_value = None
if self.error_message:
if self.continue_after_error:
return_value = 0
else:
return_value = 1 #1 means shut down this module
return return_value
def test_error_handler(self):
try: #Intentionally throw an error to test error handling
i = 4
c = "yes"
ic = i + c
except:
success = False
self.abort_all = False
self.error_message = "Test failed."
self.continue_after_error = True
exc_type, exc_value, exc_traceback = sys.exc_info() #sys.exc_inf() returns a 3-item tuple with exception information
print("\n\nGMS Exception in DataDict module... %s %s %s" % (exc_type, exc_value, exc_traceback) ) #
self.logobject.logit("\n GMS DataDict Error Line: %s" % (exc_traceback.tb_lineno), True, True ) #
tb_index = 0
tb_list = traceback.extract_tb(exc_traceback, 4)
for tb in tb_list:
print("In DataDict, exc_traceback " + str(tb_index) + ": " + str(tb))
for segment in tb:
print("# %s" % (segment) )
tb_index += 1
self.logobject.logit("\nGMS Calling handle_error with message '%s'" % (self.error_message) ) #
self.controller.handle_error(self.error_message, exc_type, exc_value, exc_traceback, self.continue_after_error, self.abort_all, self.class_alias) #Allow the Bigmatch Controller (main.py) to deal with the error, based on continue_after_error and abort_all.
print("\nEnd of Try-Except block in DataDict")
finally:
pass
class DataDict_Model():
debug = False
error_message = None
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
datadict_file_to_load_from = None #Data dictionary file name and path, which will be loaded into the entry grid if user requests.
#datadict_loadfile_displaybox = None #A textbox where the data dict file to be loaded will be displayed
datadict_file_to_save_to = None #Data dictionary file name and path, to which the metadata will be saved if user requests.
#datadict_savefile_displaybox = None #A textbox where the save-to data dict file will be displayed
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
title = '' #Title to be displayed in the Frame object
mem_or_rec = None #Is this datadict associated with a Record File or a Memory File?
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
#Arrays to hold the contents of the Data Dictionary:
meta_columns = None #List of column names within the data dictionary CSV (column_name, column_width, etc.)
meta_rownums = None #For convenience, a simple List of integers that will be displayed vertically on the left side of the Data Dict grid display.
meta_values = None #List of values for each cell within the data dictionary CSV (actually a list of lists - a list of "rows" from the data dictionary CSV)
meta_values_after_edit = [
] #List of values from the Entry Grid, AFTER the user has modified the values and clicked SAVE.
btnLoadDictFile = None
btnSaveToDictFile = None
def __init__(
self,
parent_window,
controller,
datadict_file_to_load_from=None,
show_view=None,
title='Data dictionary',
**kw
): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
#Frame.__init__(self, parent_window)
self.parent_window = parent_window #parent_wiondow is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
self.title = title
self.show_view = show_view
self.datadict_file_to_load_from = datadict_file_to_load_from
self.logobject = CHLog()
self.logobject.logit(
"\n" + "In DataDict._init_, datadict_file_to_load_from = '" +
str(self.datadict_file_to_load_from) + "' -- and show_view=" +
str(self.show_view), True, True)
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor = gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width = gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height = gl_frame_height
if self.check_key_exists("mem_or_rec", **kw):
self.mem_or_rec = str(kw["mem_or_rec"]).lower().strip()
def read_datadict_file_into_arrays(self, file_name):
'''A data dictionary CSV file is read into arrays (which are properties of this Model object). From the arrays they can be written back to a CSV file later. '''
self.logobject.logit(
"\n" +
"In DataDict.read_datadict_file_into_arrays, datadict_file_to_load_from = '"
+ str(file_name) + "'", True, True)
self.meta_columns = []
self.meta_rownums = []
self.meta_values = []
datadict_arrays = {}
f = open(file_name, "r")
content = f.readlines()
i = 0
for line in content:
print(line)
if (i == 0): #Header row in the CSV file
self.meta_columns = line.split(",")
else:
meta_temp = line.split(
","
) #meta_temp is a LIST consisting of one row from the data dictionary
self.meta_values.append(
meta_temp
) #meta_values is a LIST of LISTS, consisting of one "outer" list representing all the rows from the data dictionary, and an "inner" list consisting of the cell values for a single row.
self.meta_rownums.append(str(i))
i += 1
def write_datadict_from_entrygrid(self, file_name=None):
if file_name is None:
file_name = self.datadict_file_to_save_to
print("\n Top of write_datadict_from_entrygrid(), file_name is '%s'" %
(file_name))
#Call the View object to retrieve Entry Grid values (after they have been update by the user).
if file_name:
del self.meta_values_after_edit[:]
self.meta_values_after_edit = [] #Clear out any previous entries
#Load the entry grid items into the meta_values_after_edit list:
self.meta_values_after_edit = self.view_object.retrieve_grid_values(
)
if self.meta_values_after_edit:
newrow = []
self.logobject.logit(
"Dict file in write_datadict_from_entrygrid(): %s ...File will be deleted if it already exists."
% (str(file_name)), True, True)
if os.path.isfile(file_name):
os.remove(
file_name
) #Should not need to delete the file, but it has been appending new text at end of file instead of erasing existing text and re-writing-- even though the file open uses "w" not "w+", and it is closed after every write session.
with open(file_name, 'w', encoding='UTF8') as csvfile:
csvwriter = csv.writer(
csvfile,
delimiter=',',
quotechar='"',
quoting=csv.QUOTE_MINIMAL,
lineterminator='\n'
) #os.linesep) #chr(13) + chr(10)
#Get a list of column names retrieved from the Entry Grid -- these are CSV header columns.
j = 0
for col in self.view_object.entry_grid.colList:
newcol = str(col).replace("\n", "").strip()
newcol = newcol.replace(chr(10), "")
newcol = newcol.replace(chr(13), "")
if (
j == 0 and newcol == ''
): #First "column name" is usually blank because it is the row-numbers column, which has no header caption
pass
else:
newrow.append(newcol)
j += 1
csvwriter.writerow(
newrow) #Write the Column Headers to the CSV file
newrow = []
i = 0
for row in self.meta_values_after_edit:
self.logobject.logit(
"\n In write_datadict(), ROW: %s (length is %s and type is %s"
% (str(i), str(len(row)), str(type(row))), True,
True)
self.logobject.logit(row, True, True)
newrow = []
j = 0
for col in row:
#if col.find("\n") != -1:
# print("Reassigning col %s in row %s to a string value," % (str(j), str(i)) )
# #row[i] = newcol #str(col)
#else:
#newcol = str(col)
newcol = str(col).replace("\n", "")
newcol = newcol.replace(chr(10), "")
newcol = newcol.replace(chr(13), "")
#print(newcol)
newrow.append(newcol)
j += 1
#print(newrow)
csvwriter.writerow(newrow)
i += 1
csvfile.close()
def load_standard_datadict_headings(self):
self.meta_columns = []
self.meta_columns = [
'column_name', 'start_pos', 'width', 'unique_id_yn',
'matchfield_yn', 'bigmatch_type', 'data_format', 'comments'
]
return self.meta_columns
def display_openfile_dialogs(self, container, default_filepath=''):
file_types = [('All files', '*'), ('Text files', '*.csv;*.txt')]
kw_fpath = {
"bgcolor": self.bgcolor,
"frame_width": "",
"frame_height": "",
"file_category": "datadict"
}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(
self.parent_window, self, self.controller,
"Data dictionary file to load:", open_or_save_as, "DataDictToLoad",
file_types, **kw_fpath)
self.filepathobj_load_from.display_view(
container) #Display the dialog for user to select a data dict file
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(
self.parent_window, self, self.controller,
"Save data dictionary to file:", open_or_save_as,
"DataDictToSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(
container
) #Display the dialog for user to Ave As... a new data dict file
def instantiate_view_object(self, container):
self.view_object = DataDict_View(container, self)
return self.view_object
def display_view(self, container=None):
#Most often, we will display the Data Dictionary form in the main class's BigCanvas.BigFrame "container".
print("In DataDict.Display_View, Type of Container is '%s'" %
(str(type(container))))
kw_dict = {
"width": self.frame_width,
"background": self.bgcolor,
"borderwidth": 2,
"padx": 3,
"pady": 3
}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
#self.controller.clear_main_canvas()
if self.datadict_file_to_load_from:
#Function "read_datadict_file_into_arrays" extracts the contents of the Data Dictionary file and stores it in arrays. The arrays are them passed to TkEntry to be displayed.
print(
"\n" +
"In DataDict.display_view(), about to run read_datadict_file_into_arrays() with '"
+ str(self.datadict_file_to_load_from) + "'")
self.read_datadict_file_into_arrays(
self.datadict_file_to_load_from)
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(
container
) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container,
self.datadict_file_to_load_from)
self.display_user_buttons(container)
print(
"\n In DataDict.display_view(), about to call view_object.initUI()."
)
#Display the VIEW for this data dictionary
self.view_object.initUI(
**kw_dict
) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
self.view_object.display_datadict_in_grid(
**kw_dict
) #When first initiated, the View object will call this during its InitUI() method. But to refresh it afterwards, call display_datadict_in_grid() explicitly.
self.controller.refresh_main_canvas()
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(
".tk"
) == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(
self.button_frame, "DataDict_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(background=self.bgcolor)
#Note: the specified CSV data dictionary file is loaded automatically when the user selects it (because update_filepath() is called )
#self.btnLoadDictFile = Button(self.button_frame, text="Load Data Dictionary from CSV File", width=30, command=self.display_view)
#self.btnLoadDictFile.grid(row=0, column=0, sticky=W)
#self.btnLoadDictFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to load
self.btnSaveToDictFile = Button(
self.button_frame,
text="Save Data Dictionary to CSV File",
width=30,
command=self.write_datadict_from_entrygrid)
self.btnSaveToDictFile.grid(row=0, column=1, sticky=W)
self.btnSaveToDictFile.config(
state=DISABLED
) #Do not enable this button unless the user has selected a Data Dictionary file to save as
def update_filepath(self,
file_name_with_path='',
callback_string='',
alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
self.logobject.logit(
"Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'"
% (file_name_with_path, callback_string), True, True)
if callback_string.lower().strip(
)[:4] == "load" or callback_string.lower().strip()[:4] == "open":
self.datadict_file_to_load_from = file_name_with_path
#if file_name_with_path: #Commented this out because btnLoadDictFile seems to be unnecessary - we automatically load the Data Dict
# self.btnLoadDictFile.config(state=NORMAL)
#Refresh the Data Dictionary view when the user selects a new DataDict file.
if self.datadict_file_to_load_from:
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid(
) #Remove all existing values from the grid
elif callback_string.lower().strip(
)[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.datadict_file_to_save_to = file_name_with_path
if self.datadict_file_to_save_to:
self.btnSaveToDictFile.config(state=NORMAL)
else:
self.btnSaveToDictFile.config(state=DISABLED)
self.update_master_paths(file_name_with_path)
def update_master_paths(self, file_name_with_path):
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
if self.mem_or_rec == "mem":
self.controller.mem_datadict_last_opened = file_name_with_path #The controller tracks last file opened of this type, so that when the user is again prompted to open the same type of file, we can default this value in.
if self.mem_or_rec == "rec":
self.controller.rec_datadict_last_opened = file_name_with_path #The controller tracks last file opened of this type, so that when the user is again prompted to open the same type of file, we can default this value in.
print(
"\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s"
% (self.controller.dir_last_opened,
self.controller.datadict_dir_last_opened,
self.controller.rec_datadict_last_opened,
self.controller.mem_datadict_last_opened))
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
class Error_UI_Model():
debug = False
error_message = None #Plain text error message
error_list = [
] #List of errors that have been logged by the controller (LIST OF DICTS - EACH DICT CONTAINS AN ERROR MESSAGE AND EXCEPTION OBJECTS)
error_dict = None #The ERROR DICT currently being processed
parent_window = None #Parent_wiondow is the TKinter object itself (often known as "root"
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
title = None
bgcolor = None
bgcolors = []
frame_width = None
frame_height = None
dedupe_single_file = False #Is this a dedupe of a single file, or a match of two different files?
#parmf_file = None
def __init__(self,
parent_window,
controller,
error_list=["Unspecified error (ui model parm)"],
title="Error",
bgcolor=gl_frame_color,
frame_width=gl_frame_width,
frame_height=gl_frame_height):
self.parent_window = parent_window #Parent_wiondow is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
self.error_list = error_list #Error_list is a list of DICTS - each dict contains a plain text error message and exception objects)
now = datetime.datetime.now()
self.init_time = str(now.year) + str(now.month).rjust(
2, '0') + str(now.day).rjust(2, '0') + ' ' + str(now.hour).rjust(
2, '0') + ':' + str(now.minute).rjust(2, '0')
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit(
"\n\n____________________________________________________________________",
True, True)
self.logobject.logit(
"%s In Error_UI_Model._init_: title=%s" % (self.init_time, title),
True, True)
if title is not None:
self.title = title
else:
self.title = "Nothing"
if frame_width is not None:
self.frame_width = frame_width
if frame_height is not None:
self.frame_height = frame_height
#if show_view is not None:
# self.show_view = show_view
if bgcolor is not None:
self.bgcolor = bgcolor
else:
self.bgcolor = gl_frame_color
self.bgcolors = [
"#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0",
"#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0"
]
if self.error_message is not None:
self.logobject.logit("\nCalling handle_error with message '%s'" %
(self.error_message))
return
#*****************************************************************************************************************************************
#ERROR VIEW OBJECT(S)
#NOTE: VIEWS are typically invoked by calling this MODEL object. The following methods are handlers for instantiating VIEWS.
def instantiate_view_object(self, container, index):
#Instantiate VIEWS here.
error_view = Error_UI_View(container, self, index)
return error_view
def display_view(self, container, index, **kw):
error_view = self.instantiate_view_object(container, index)
print("Kwargs for error_view.display_view(): ")
self.logobject.logit(str(kw), True, True)
for key, value in kw.items():
self.logobject.logit("%s = %s" % (key, value), True, True)
self.logobject.logit(
"\n In error_ui_model.display_view(), calling error_view.initUI().",
True, True)
error_view.initUI(**kw) #DISPLAY THE FRAME OBJECT ON SCREEN
return error_view
def display_views(self, container=None, howmany_views=None):
#Instantiate VIEWS here.
print("\n Top of error_view.display_views()")
if container is None:
container = self.controller.bigcanvas.bigframe
if not self.error_list:
self.error_list = ["Unspecified error (ui model display views)"]
if howmany_views is None:
howmany_views = len(self.error_list)
self.logobject.logit("\n Top of error_view.display_views()", True,
True)
if True: #to do: validate the existence of error message before launching UI
for i in range(0, howmany_views):
bgcolor = self.bgcolors[i] #bgcolor = "#FDFFDF"
print(
"\n In Error_UI_Model.display_views(), calling display_view() for iteration #"
+ str(i) + ". BgColor=" + bgcolor)
#DISPLAY THE ERROR DISPLAY SCREEN HERE:
self.display_view(container,
i,
width=self.frame_width,
background=bgcolor,
borderwidth=2,
padx=3,
pady=3)
else:
pass #
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(
".tk"
) == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(
self.button_frame, "error_view.display_user_buttons()")
else:
stackslot = 0
#Testing: show a button that will dump to the command window the current contents of all the screen CONTROLS.
self.button_frame.grid(row=stackslot, column=0, sticky=EW)
self.button_frame.config(background=self.bgcolor)
def update_message_region(self, text='', clear_after_ms=4000, **kw):
#if not text:
# text = "Uh-oh"
self.message_region.configure(text=text)
self.message_region.after(clear_after_ms, self.clear_message_region)
def clear_message_region(self):
self.message_region.configure(text="")
class BigMatchController():
'''BigMatchController class creates a menu and a canvas for displaying various widgets. It also calls all methods necessary for instantiating those various widgets.'''
debug = False
debug_global = None #Debug settings in all child modules canbe turned on or off with this one controller property
error_message = None
error_list = [] #Collection of errors that have been logged
parent_window = None
bigcanvas = None #Main scrolling canvas on which the main frame object sits. The canvas serves mainly to host the scrollbars.
framestack_counter = None
#Objects that represent major functions of this application (Blocking Pass, Data Dictionary, etc.)
blockingpass_model = None
blocking_passes = [] #Index of Blocking Pass objects that have been instantiated
how_many_blk_passes = 10
matchreview_model = None
error_ui = None #Frame for displaying error information at runtime
datadict_model = None
dir_last_opened = None
datadict_dir_last_opened = None
blockingpass_dir_last_opened = None
parmfile_dir_last_opened = None
resultfile_last_opened = None
convertfile_last_opened = None
rec_datadict_last_opened = None
mem_datadict_last_opened = None
parmf_last_opened = None
parmn_last_opened = None
match_result_dir_last_opened = None
logobject = None
user = None
common = None
host_name = None
bigmatch_exe_location = None
enable_logging = None #Shut down logging until we solve permissions issues on Pennhurst server
enable_sqlite = None
sqlite_found = None
sas7bdat_found = None
python_common_found = None
os_name = None
os_platform = None
os_release = None
uid = None
gs = None
def __init__(self, parent_window):
self.parent_window = parent_window
self.parent_window.title("BigMatch configuration") #parent window title
self.parent_window.columnconfigure(0, weight=0, pad=3)
self.parent_window.rowconfigure(0, weight=0, pad=3)
self.host_name = str(socket.gethostname()).lower().strip()
self.uid = getpass.getuser().lower().strip()
print("\n--------------------------------------------------------------------------------")
print("\n STARTING NEW SESSION. hostname: %s, user: %s" % (self.host_name, getpass.getuser() ) )
#Get operating system info
self.os_name = os.name
self.os_platform = platform.system()
self.os_release = platform.release()
self.python_common_found = python_common_found
self.sas7bdat_found = sas7bdat_found
self.sqlite_found = sqlite_found
print("\nPython_common_found? %s" % (self.python_common_found) )
print("\n os_name: %s, os_platform: %s, os_release: %s" % (self.os_name, self.os_platform, self.os_release) )
if (self.host_name == "chrsd1-gsanders" and self.uid == "gsanders"):
self.enable_logging = True
self.gs = True
elif (self.host_name == "greg-hp" and self.uid == "greg") or (self.host_name == "gregory" and self.uid == "lisa"):
self.enable_logging = True
if self.enable_logging:
#"Enable logging" does not mean the user has the necessary file/folder permissions. But from the GUI rules standpoint, this user can write logs if the permissions are in place.
print("Logging enabled (by user ID)")
#Make sure we can actually write to a file (file/folder permissions support logging)
self.logobject = CHLog(self.enable_logging, self.enable_logging)
logtest = self.logobject.test_subfolder_filewrite()
print("Logging write test returned %s)" % (logtest) )
self.enable_logging = logtest #If test failed, set enable_logging to False
if not self.enable_logging:
print("Logging write test failed")
self.logobject = None
if self.sqlite_found:
self.enable_sqlite = self.logobject.test_sqlite_data_write()
if self.enable_logging:
self.user = CHUser()
self.common = CommonRL(self.parent_window, self)
self.bigmatch_exe_location = self.get_bigmatch_exe_location()
head, tail = os.path.split(os.path.abspath(__file__))
#self.update_controller_dirpaths(head) #No need to set this to a default location at the outset. That default-setting is handled by the individual FilePath objects.
## Grid sizing behavior in window
self.parent_window.grid_rowconfigure(0, weight=1)
self.parent_window.grid_columnconfigure(0, weight=1)
#Add a Canvas to the parent window:
self.bigcanvas = ScrollCanvas(parent_window) #BigCanvas is the main container which holds bigfraame, which everything else
#******************************************************
#INTRO SCREEN:
self.load_splash("Chapin Hall's BigMatch user interface - blocking pass definitions")
#******************************************************
#Disable the following section. OpenFile dialogs are managed by individual modules such as datadict, blockingpass, etc.
#FILE LOCATION FRAMES:
#Parms: FilePath(parent_window, show_view=False, title='', bgcolor=gl_frame_color, file_types=[('All files', '*'),('Text files', '*.csv;*.txt')], frame_width=gl_frame_width, frame_height=gl_frame_height)
#frame_color = "ivory"
#open_or_save_as = "open"
#self.filepathobj_memfile_dict.display_view(self.bigcanvas.bigframe) #Display the dialog for user to select a data dict file
#*******************************************************************************************************************
#MENU OPTIONS
menubar = Menu(self.parent_window)
self.parent_window.config(menu=menubar)
dictMenu = Menu(menubar, tearoff=0)
dictMenu.add_command(label="Create or edit data dictionary", command=self.load_datadict)
#dictMenu.add_command(label="Create or edit data dictionary for the Record File", command=self.load_recfile_datadict)
#dictMenu.add_command(label="Create or edit data dictionary for the Memory File", command=self.load_memfile_datadict)
dictMenu.add_command(label="Start over with new data dictionary", command=self.startclean_new_datadict)
menubar.add_cascade(label="Data dictionaries", menu=dictMenu)
blockMenu = Menu(menubar, tearoff=0)
blockMenu.add_command(label="Create parameter file from blocking passes", command=self.load_blocking_passes)
blockMenu.add_command(label="Start over with new blocking pass", command=self.startclean_new_blockingpass)
menubar.add_cascade(label="Blocking passes", menu=blockMenu)
reviewMenu = Menu(menubar, tearoff=0)
reviewMenu.add_command(label="Review match results", command=self.load_match_review)
#reviewMenu.add_command(label="Combine reviewed results", command=self.load_combine_reviewed_results)
menubar.add_cascade(label="Match results", menu=reviewMenu)
'''def get_command_for_file_convert_sas_to_text():
cmd_string = "self.load_convert_file('sas', 'text')"
print("\nReturning the string %s" % (cmd_string) )
return cmd_string'''
if self.python_common_found:
#if self.uid.find("sanders") > -1:
convertMenu = Menu(menubar, tearoff=0)
if sas7bdat_found:
convertMenu.add_command(label="Convert SAS file to text", command=self.load_convert_file_sas_to_text)
convertMenu.add_command(label="Convert CSV file to flat file", command=self.load_convert_file_csv_to_text)
convertMenu.add_command(label="Convert csv or flat file to sqlite table", command=self.load_convert_file_txt_to_sqlite)
menubar.add_cascade(label="Convert files", menu=convertMenu)
if self.sqlite_found:
#if self.uid.find("sanders") > -1:
dbMenu = Menu(menubar, tearoff=0)
dbMenu.add_command(label="Display Sqlite data", command=self.display_sqlite_data)
menubar.add_cascade(label="Read database", menu=dbMenu)
if self.enable_logging:
defltMenu = Menu(menubar, tearoff=0)
defltMenu.add_command(label="Set startup to Data Dictionary", command=self.set_startup_module_to_data_dict)
defltMenu.add_command(label="Set startup to Blocking Pass", command=self.set_startup_module_to_blocking_pass)
defltMenu.add_command(label="Set startup to Match Review", command=self.set_startup_module_to_match_review)
defltMenu.add_command(label="Set startup to File Conversion", command=self.set_startup_module_to_file_convert)
menubar.add_cascade(label="Default settings", menu=defltMenu)
#Test error display:
self.add_error_to_list("Testing 123", "Type exception", "Hey, you can't do that!", None)
self.add_error_to_list("Another Test", "Format exception", "No, you can't do that either.", None)
errMenu = Menu(menubar, tearoff=0)
errMenu.add_command(label="View errors", command=self.load_error_display)
menubar.add_cascade(label="Error display", menu=errMenu)
#if self.bigmatch_exe_location:
# runMenu = Menu(menubar, tearoff=0)
# runMenu.add_command(label="Run BigMatch", command=self.generate_parmfile)
# menubar.add_cascade(label="Run BigMatch", menu=runMenu)
#genMenu = Menu(menubar, tearoff=0)
#genMenu.add_command(label="Create parameter file", command=self.generate_parmfile)
#genMenu.add_command(label="Load ParmF file", command=self.load_blocking_pass_from_parmfile)
#menubar.add_cascade(label="Generate", menu=genMenu)
#otherMenu = Menu(menubar, tearoff=0)
#otherMenu.add_command(label="Clear display", command=self.bigcanvas.bigframe.clear_canvas)
#otherMenu.add_command(label="Restore display", command=self.bigcanvas.bigframe.clear_canvas)
#menubar.add_cascade(label="Clear", menu=otherMenu)
#End of MENU OPTIONS
#*************************************************************
#Load the user's preferred start-up module:
if self.enable_logging:
setting = self.user.get_config_setting("cmd_onload")
if setting.lower().strip() == "load_datadict":
self.load_datadict()
elif setting.lower().strip() == "load_blocking_passes":
self.load_blocking_passes()
elif setting.lower().strip() == "load_match_review":
self.load_match_review()
elif setting.lower().strip() == "load_convert_file":
self.load_convert_file_csv_to_text()
else:
self.load_datadict()
else:
self.load_datadict()
def update_controller_dirpaths(self, file_name_with_path): #Set default locations for FileOpen dialogs at session launch.
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.dir_last_opened = head
self.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
self.parmfile_dir_last_opened = head
self.match_result_dir_last_opened = head
print("\n _MAIN_saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s" % (self.dir_last_opened, self.datadict_dir_last_opened, self.rec_datadict_last_opened, self.mem_datadict_last_opened) )
def load_splash(self, caption="Chapin Hall's BigMatch utilities"):
self.splash_frame = Frame(self.bigcanvas.bigframe)
stackslot = self.bigcanvas.bigframe.get_widget_position(self.splash_frame, "main.load_splash()")
self.splash_frame.grid(row=stackslot, column=0, columnspan=4, sticky=EW)
self.splash_frame.config(background="ivory", width=130, padx=0, pady=0)
self.splash_label = Label(self.splash_frame, text=caption)
self.splash_label.grid(row=0, column=0, sticky=W)
self.splash_label.config(font=("Arial", 16, "bold"), borderwidth=1, width=65, anchor=W, justify="left", padx=0, pady=0, background="ivory") #width=100,
self.bigcanvas.bigframe.refresh_canvas()
def load_datadict(self, datadict_filename="", mem_or_rec="rec"):
print("In main.load_datadict() with mem_or_rec=%s and datadict_filename=%s" % (str(mem_or_rec), str(datadict_filename) ) )
show_frame = False #Don't immediately display the view
if self.datadict_model:
self.datadict_model = None
kw_datadict = {"mem_or_rec":mem_or_rec}
self.bigcanvas.bigframe.clear_canvas() #Unload (hide) all frame objects
self.load_splash("BigMatch data dictionary manager") #dictionary for " + which_name)
self.datadict_model = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Record file data dictionary", **kw_datadict)
self.datadict_model.display_view(self.bigcanvas.bigframe)
return True
'''
def load_datadict(self, datadict_filename="", mem_or_rec="rec"):
print("In main.load_datadict() with mem_or_rec=%s and datadict_filename=%s" % (str(mem_or_rec), str(datadict_filename) ) )
if which.lower()=="rec":
which_name = "Record File"
elif which.lower()=="mem":
which_name = "Memory File"
else:
self.error_message = "Invalid data dictionary type: " + which
print(self.error_message)
self.bigcanvas.bigframe.clear_canvas() #Unload (hide) all frame objects
self.load_splash("BigMatch data dictionary manager") #dictionary for " + which_name)
self.datadictobj.display_view(self.bigcanvas.bigframe)
return True
def load_recfile_datadict(self, datadict_filename=""):
show_frame = False #Don't immediately display the view
#if not self.datadictobj_recfile:
# self.datadictobj_recfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Record file data dictionary")
if self.datadictobj_recfile:
self.datadictobj_recfile = None
kw_datadict = {"mem_or_rec":"rec"}
self.datadictobj_recfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Record file data dictionary", **kw_datadict)
#if datadict_filename: #If a filename was passed to this function, store it to the DataDict object's "datadict_filename" property.
# self.datadictobj_recfile.datadict_filename = datadict_filename
result = self.load_datadict("rec", datadict_filename)
return result
def load_memfile_datadict(self, datadict_filename=""):
show_frame = False #Don't immediately display the view
#if not self.datadictobj_memfile:
# self.datadictobj_memfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Memory file data dictionary")
if self.datadictobj_memfile:
self.datadictobj_memfile = None
kw_datadict = {"mem_or_rec":"mem"}
self.datadictobj_memfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Memory file data dictionary", **kw_datadict)
#if datadict_filename: #If a filename was passed to this function, store it to the DataDict object's "datadict_filename" property.
# self.datadictobj_memfile.datadict_filename = datadict_filename
result = self.load_datadict("mem", datadict_filename)
return result
'''
def load_blocking_passes(self):
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- define blocking passes")
self.blockingpass_model = BlockingPass_Model(self.parent_window, self)
self.blockingpass_model.display_views(self.bigcanvas.bigframe, self.how_many_blk_passes)
'''def load_blocking_pass_from_parmfile(self):
parmfile = os.path.join('C:\Greg', 'code', 'bigmatch_utilities', 'BigMatchGui', 'parmf.txt')
parmfileobj = BigmatchParmfile(parmfile)
for parm in parmfileobj.parms:
print("PARMFILE PARM: blkpass: %s, row_index: %s, row_type: %s, parms in row: %s, parm_index: %s, parm_type: %s, parm_value: %s" % (parm["blocking_pass"], parm["row_index"], parm["row_type"], parm["parms_in_row"], parm["parm_index"], parm["parm_type"], parm["parm_value"] ) )
'''
def load_match_review(self):
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- review match results")
self.matchreview_model = MatchReview_Model(self.parent_window, self)
#self.matchreview_model.display_view(self.bigcanvas.bigframe)
self.matchreview_model.display_views(self.bigcanvas.bigframe, 1)
def load_combine_reviewed_results(self):
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- combine reviewed match results")
self.matchreview_model = MatchReview_Model(self.parent_window, self)
#self.matchreview_model.display_view(self.bigcanvas.bigframe)
self.matchreview_model.display_views(self.bigcanvas.bigframe, 1)
def load_error_display(self):
print("In main.load_error_display()")
self.bigcanvas.bigframe.clear_canvas() #Unload (hide) all frame objects
self.load_splash("BigMatch utilities - error report")
if not self.error_list:
self.error_list = ["Unspecified error (main)"]
self.error_ui = Error_UI_Model(self.parent_window, self, self.error_list)
self.error_ui.display_views(self.bigcanvas.bigframe, len(self.error_list))
print("\n\nBack in MAIN.PY after error was displayed. \n\n")
'''def load_convert_file(self, source_format="sas", output_format="text"):
print("\nIn MAIN, load_convert_file() with source %s and output %s" % (source_format, output_format) )
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- convert files")
self.convertfile_model = ConvertFile_Model(self.parent_window, self, source_format, output_format)
self.convertfile_model.display_view(self.bigcanvas.bigframe)'''
def instantiate_convert_file(self, source_format=None, output_format=None):
print("\nIn MAIN, load_convert_file() with source %s and output %s" % (source_format, output_format) )
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- convert files")
self.convertfile_model = ConvertFile_Model(self.parent_window, self, source_format, output_format)
self.convertfile_model.display_view(self.bigcanvas.bigframe)
def instantiate_datafile_rdbms_ui(self, source_format="txt", output_format="sqlite"):
print("\nIn MAIN, load datafile_to_rdbms_ui()")
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- convert text file to database table")
self.convertfile_model = Datafile_to_RDBMS_UI_Model(self.parent_window, self, source_format, output_format)
self.convertfile_model.display_view(self.bigcanvas.bigframe)
def instantiate_readexport_rdbms_ui(self, db_platform="sqlite", output_format="text"):
print("\nIn MAIN, load ReadExport_Rdbms_UI()")
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- read/export data from database")
self.db_readexport_model = RDBMS_Read_Export_UI_Model(self.parent_window, self, db_platform, output_format)
self.db_readexport_model.display_view(self.bigcanvas.bigframe)
def display_sqlite_data(self):
self.instantiate_readexport_rdbms_ui("sqlite")
def load_convert_file_sas_to_text(self):
self.instantiate_convert_file("sas", "text")
#self.convertfile_model.set_source_format("sas")
#self.convertfile_model.set_output_format("text")
def load_convert_file_csv_to_text(self):
self.instantiate_convert_file("csv", "text")
def load_convert_file_txt_to_sqlite(self):
self.instantiate_datafile_rdbms_ui("txt", "sqlite")
def generate_parmfile(self, filename_with_path=''):
print("This function is not yet available.")
def clear_canvas(self):
'''clear_canvas() calls the function of the same name, within the child frame within this object's child scrolling frame.
This is used when the user switches between Data Dictionary and Blocking Pass screens, or similar switch.'''
self.bigcanvas.bigframe.clear_canvas()
def refresh_main_canvas(self):
self.bigcanvas.bigframe.update_idletasks()
self.bigcanvas.update_idletasks()
self.bigcanvas.config(scrollregion=self.bigcanvas.bbox(ALL)) #Canvas object needs to encompass new items
def set_startup_module_to_blocking_pass(self):
self.user.write_setting_to_config_file('cmd_onload', 'load_blocking_passes')
def set_startup_module_to_match_review(self):
self.user.write_setting_to_config_file('cmd_onload', 'load_match_review')
def set_startup_module_to_file_convert(self):
self.user.write_setting_to_config_file('cmd_onload', 'load_convert_file_csv_to_text')
def set_startup_module_to_data_dict(self):
self.user.write_setting_to_config_file('cmd_onload', 'load_datadict')
def startclean_new_datadict(self):
self.clear_canvas()
self.redisplay_module_after_error("datadict")
def startclean_new_blockingpass(self):
self.clear_canvas()
self.redisplay_module_after_error("blockingpass")
def redisplay_module_after_error(self, module_name="errordisplay"):
'''Note: This function might be executed after an error, but it might be initiated by the user if they want to wipe out what they started, and start fresh. '''
print("\n Back in main.py after error, calling module %s" % (module_name) )
if str(module_name).lower().strip() == "datadict":
self.load_datadict()
elif str(module_name).lower().strip() == "blockingpass":
self.load_blocking_passes()
elif str(module_name).lower().strip() == "matchreview":
self.load_match_review()
elif str(module_name).lower().strip() == "convertfile":
self.load_convert_file_csv_to_text()
elif str(module_name).lower().strip() == "errordisplay":
self.load_error_display()
else:
self.load_error_display()
print("\nEnd of Main.redisplay_module_after_error()")
def kill_module(self, module_name):
'''Set to nothing the object that is to be destroyed. This assumes that this BigMatch Controller holds the only reference to that object (which is most often the case)'''
print("\n Main.py kill_module(), killing module %s" % (module_name) )
if str(module_name).lower().strip() == "datadict":
self.datadict_model = None
elif str(module_name).lower().strip() == "blockingpass":
self.blockingpass_model = None
elif str(module_name).lower().strip() == "matchreview":
self.matchreview_model = None
elif str(module_name).lower().strip() == "convertfile":
self.convertfile_model = None
print("\nEnd of Main.kill_module()")
def get_bigmatch_exe_location(self):
if self.host_name.lower() == "pennhurst1.chapinhall.org":
loc = os.path.join("/usr", "local", "bin", "bigmatch")
self.bigmatch_exe_location = loc
elif self.host_name.lower() == "chrsd1-gsanders":
loc = os.path.join("C:\Greg", "ChapinHall", "RecordLinking", "BigMatch", "bigmatchqst.exe")
self.bigmatch_exe_location = loc
else:
self.bigmatch_exe_location = "bigmatch"
return self.bigmatch_exe_location
def call_bigmatch(self):
#subprocess.call(args, *, stdin=None, stdout=None, stderr=None, shell=False)
process = subprocess.Popen(self.bigmatch_exe_location, stdout=subprocess.PIPE, creationflags=0x08000000)
process.wait()
def get_command_for_file_convert_sas_to_text(self):
cmd_string = "self.load_convert_file('sas', 'text')"
print("\nReturning the string %s" % (cmd_string) )
return cmd_string
def handle_error(self, error_message='Error. Procedure cancelled.', excp_type="", excp_value="", excp_traceback="", continue_after_error=False, abort_all=False, calling_object_name=None, module_to_reload_afterwards="errordisplay"):
print("\nTop of Main.handle_error(), error_message: %s, excp_type: %s, excp_value: %s, continue_after: %s" % (error_message, excp_type, excp_value, continue_after_error))
self.error_message = error_message
self.add_error_to_list(self.error_message, excp_type, excp_value, excp_traceback)
if self.logobject:
self.logobject.logit("\nCalling Common.handle_error with message '%s'" % (self.error_message), True, True ) #Log this error in disk log and/or database
self.common.handle_error(self.error_message, excp_type, excp_value, excp_traceback, continue_after_error, abort_all) #To make the error handler more generic, place it in a lightweight common functions lib so that it can be called even when this BigMatch controller is not in use.
if abort_all:
sys.exit(1) #Shut down the Python interpreter
if not continue_after_error:
if calling_object_name: #Whatever class/object called this error handler is unceremoniously dumped as a result
self.kill_module(calling_object_name)
self.clear_canvas()
self.redisplay_module_after_error(module_to_reload_afterwards)
print("\n\nEnd of Main.handle_error()")
def add_error_to_list(self, error_message="Unspecified error (in main)", excp_type="", excp_value="", excp_traceback=""):
errdict = {"error_message":error_message, "exc_type":excp_type, "exc_value":excp_value, "exc_traceback":excp_traceback}
self.error_list.append(errdict)
return len(self.error_list)
class BigMatchController():
'''BigMatchController class creates a menu and a canvas for displaying various widgets. It also calls all methods necessary for instantiating those various widgets.'''
debug = False
debug_global = None #Debug settings in all child modules canbe turned on or off with this one controller property
error_message = None
error_list = [] #Collection of errors that have been logged
parent_window = None
bigcanvas = None #Main scrolling canvas on which the main frame object sits. The canvas serves mainly to host the scrollbars.
framestack_counter = None
#Objects that represent major functions of this application (Blocking Pass, Data Dictionary, etc.)
blockingpass_model = None
blocking_passes = [
] #Index of Blocking Pass objects that have been instantiated
how_many_blk_passes = 10
matchreview_model = None
error_ui = None #Frame for displaying error information at runtime
datadict_model = None
dir_last_opened = None
datadict_dir_last_opened = None
blockingpass_dir_last_opened = None
parmfile_dir_last_opened = None
resultfile_last_opened = None
convertfile_last_opened = None
rec_datadict_last_opened = None
mem_datadict_last_opened = None
parmf_last_opened = None
parmn_last_opened = None
match_result_dir_last_opened = None
logobject = None
user = None
common = None
host_name = None
bigmatch_exe_location = None
enable_logging = None #Shut down logging until we solve permissions issues on Pennhurst server
enable_sqlite = None
sqlite_found = None
sas7bdat_found = None
python_common_found = None
os_name = None
os_platform = None
os_release = None
uid = None
gs = None
def __init__(self, parent_window):
self.parent_window = parent_window
self.parent_window.title(
"BigMatch configuration") #parent window title
self.parent_window.columnconfigure(0, weight=0, pad=3)
self.parent_window.rowconfigure(0, weight=0, pad=3)
self.host_name = str(socket.gethostname()).lower().strip()
self.uid = getpass.getuser().lower().strip()
print(
"\n--------------------------------------------------------------------------------"
)
print("\n STARTING NEW SESSION. hostname: %s, user: %s" %
(self.host_name, getpass.getuser()))
#Get operating system info
self.os_name = os.name
self.os_platform = platform.system()
self.os_release = platform.release()
self.python_common_found = python_common_found
self.sas7bdat_found = sas7bdat_found
self.sqlite_found = sqlite_found
print("\nPython_common_found? %s" % (self.python_common_found))
print("\n os_name: %s, os_platform: %s, os_release: %s" %
(self.os_name, self.os_platform, self.os_release))
if (self.host_name == "chrsd1-gsanders" and self.uid == "gsanders"):
self.enable_logging = True
self.gs = True
elif (self.host_name == "greg-hp"
and self.uid == "greg") or (self.host_name == "gregory"
and self.uid == "lisa"):
self.enable_logging = True
if self.enable_logging:
#"Enable logging" does not mean the user has the necessary file/folder permissions. But from the GUI rules standpoint, this user can write logs if the permissions are in place.
print("Logging enabled (by user ID)")
#Make sure we can actually write to a file (file/folder permissions support logging)
self.logobject = CHLog(self.enable_logging, self.enable_logging)
logtest = self.logobject.test_subfolder_filewrite()
print("Logging write test returned %s)" % (logtest))
self.enable_logging = logtest #If test failed, set enable_logging to False
if not self.enable_logging:
print("Logging write test failed")
self.logobject = None
if self.sqlite_found:
self.enable_sqlite = self.logobject.test_sqlite_data_write()
if self.enable_logging:
self.user = CHUser()
self.common = CommonRL(self.parent_window, self)
self.bigmatch_exe_location = self.get_bigmatch_exe_location()
head, tail = os.path.split(os.path.abspath(__file__))
#self.update_controller_dirpaths(head) #No need to set this to a default location at the outset. That default-setting is handled by the individual FilePath objects.
## Grid sizing behavior in window
self.parent_window.grid_rowconfigure(0, weight=1)
self.parent_window.grid_columnconfigure(0, weight=1)
#Add a Canvas to the parent window:
self.bigcanvas = ScrollCanvas(
parent_window
) #BigCanvas is the main container which holds bigfraame, which everything else
#******************************************************
#INTRO SCREEN:
self.load_splash(
"Chapin Hall's BigMatch user interface - blocking pass definitions"
)
#******************************************************
#Disable the following section. OpenFile dialogs are managed by individual modules such as datadict, blockingpass, etc.
#FILE LOCATION FRAMES:
#Parms: FilePath(parent_window, show_view=False, title='', bgcolor=gl_frame_color, file_types=[('All files', '*'),('Text files', '*.csv;*.txt')], frame_width=gl_frame_width, frame_height=gl_frame_height)
#frame_color = "ivory"
#open_or_save_as = "open"
#self.filepathobj_memfile_dict.display_view(self.bigcanvas.bigframe) #Display the dialog for user to select a data dict file
#*******************************************************************************************************************
#MENU OPTIONS
menubar = Menu(self.parent_window)
self.parent_window.config(menu=menubar)
dictMenu = Menu(menubar, tearoff=0)
dictMenu.add_command(label="Create or edit data dictionary",
command=self.load_datadict)
#dictMenu.add_command(label="Create or edit data dictionary for the Record File", command=self.load_recfile_datadict)
#dictMenu.add_command(label="Create or edit data dictionary for the Memory File", command=self.load_memfile_datadict)
dictMenu.add_command(label="Start over with new data dictionary",
command=self.startclean_new_datadict)
menubar.add_cascade(label="Data dictionaries", menu=dictMenu)
blockMenu = Menu(menubar, tearoff=0)
blockMenu.add_command(
label="Create parameter file from blocking passes",
command=self.load_blocking_passes)
blockMenu.add_command(label="Start over with new blocking pass",
command=self.startclean_new_blockingpass)
menubar.add_cascade(label="Blocking passes", menu=blockMenu)
reviewMenu = Menu(menubar, tearoff=0)
reviewMenu.add_command(label="Review match results",
command=self.load_match_review)
#reviewMenu.add_command(label="Combine reviewed results", command=self.load_combine_reviewed_results)
menubar.add_cascade(label="Match results", menu=reviewMenu)
'''def get_command_for_file_convert_sas_to_text():
cmd_string = "self.load_convert_file('sas', 'text')"
print("\nReturning the string %s" % (cmd_string) )
return cmd_string'''
if self.python_common_found:
#if self.uid.find("sanders") > -1:
convertMenu = Menu(menubar, tearoff=0)
if sas7bdat_found:
convertMenu.add_command(
label="Convert SAS file to text",
command=self.load_convert_file_sas_to_text)
convertMenu.add_command(label="Convert CSV file to flat file",
command=self.load_convert_file_csv_to_text)
convertMenu.add_command(
label="Convert csv or flat file to sqlite table",
command=self.load_convert_file_txt_to_sqlite)
menubar.add_cascade(label="Convert files", menu=convertMenu)
if self.sqlite_found:
#if self.uid.find("sanders") > -1:
dbMenu = Menu(menubar, tearoff=0)
dbMenu.add_command(label="Display Sqlite data",
command=self.display_sqlite_data)
menubar.add_cascade(label="Read database", menu=dbMenu)
if self.enable_logging:
defltMenu = Menu(menubar, tearoff=0)
defltMenu.add_command(label="Set startup to Data Dictionary",
command=self.set_startup_module_to_data_dict)
defltMenu.add_command(
label="Set startup to Blocking Pass",
command=self.set_startup_module_to_blocking_pass)
defltMenu.add_command(
label="Set startup to Match Review",
command=self.set_startup_module_to_match_review)
defltMenu.add_command(
label="Set startup to File Conversion",
command=self.set_startup_module_to_file_convert)
menubar.add_cascade(label="Default settings", menu=defltMenu)
#Test error display:
self.add_error_to_list("Testing 123", "Type exception",
"Hey, you can't do that!", None)
self.add_error_to_list("Another Test", "Format exception",
"No, you can't do that either.", None)
errMenu = Menu(menubar, tearoff=0)
errMenu.add_command(label="View errors",
command=self.load_error_display)
menubar.add_cascade(label="Error display", menu=errMenu)
#if self.bigmatch_exe_location:
# runMenu = Menu(menubar, tearoff=0)
# runMenu.add_command(label="Run BigMatch", command=self.generate_parmfile)
# menubar.add_cascade(label="Run BigMatch", menu=runMenu)
#genMenu = Menu(menubar, tearoff=0)
#genMenu.add_command(label="Create parameter file", command=self.generate_parmfile)
#genMenu.add_command(label="Load ParmF file", command=self.load_blocking_pass_from_parmfile)
#menubar.add_cascade(label="Generate", menu=genMenu)
#otherMenu = Menu(menubar, tearoff=0)
#otherMenu.add_command(label="Clear display", command=self.bigcanvas.bigframe.clear_canvas)
#otherMenu.add_command(label="Restore display", command=self.bigcanvas.bigframe.clear_canvas)
#menubar.add_cascade(label="Clear", menu=otherMenu)
#End of MENU OPTIONS
#*************************************************************
#Load the user's preferred start-up module:
if self.enable_logging:
setting = self.user.get_config_setting("cmd_onload")
if setting.lower().strip() == "load_datadict":
self.load_datadict()
elif setting.lower().strip() == "load_blocking_passes":
self.load_blocking_passes()
elif setting.lower().strip() == "load_match_review":
self.load_match_review()
elif setting.lower().strip() == "load_convert_file":
self.load_convert_file_csv_to_text()
else:
self.load_datadict()
else:
self.load_datadict()
def update_controller_dirpaths(
self, file_name_with_path
): #Set default locations for FileOpen dialogs at session launch.
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.dir_last_opened = head
self.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
self.parmfile_dir_last_opened = head
self.match_result_dir_last_opened = head
print(
"\n _MAIN_saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s"
% (self.dir_last_opened, self.datadict_dir_last_opened,
self.rec_datadict_last_opened, self.mem_datadict_last_opened))
def load_splash(self, caption="Chapin Hall's BigMatch utilities"):
self.splash_frame = Frame(self.bigcanvas.bigframe)
stackslot = self.bigcanvas.bigframe.get_widget_position(
self.splash_frame, "main.load_splash()")
self.splash_frame.grid(row=stackslot,
column=0,
columnspan=4,
sticky=EW)
self.splash_frame.config(background="ivory", width=130, padx=0, pady=0)
self.splash_label = Label(self.splash_frame, text=caption)
self.splash_label.grid(row=0, column=0, sticky=W)
self.splash_label.config(font=("Arial", 16, "bold"),
borderwidth=1,
width=65,
anchor=W,
justify="left",
padx=0,
pady=0,
background="ivory") #width=100,
self.bigcanvas.bigframe.refresh_canvas()
def load_datadict(self, datadict_filename="", mem_or_rec="rec"):
print(
"In main.load_datadict() with mem_or_rec=%s and datadict_filename=%s"
% (str(mem_or_rec), str(datadict_filename)))
show_frame = False #Don't immediately display the view
if self.datadict_model:
self.datadict_model = None
kw_datadict = {"mem_or_rec": mem_or_rec}
self.bigcanvas.bigframe.clear_canvas(
) #Unload (hide) all frame objects
self.load_splash("BigMatch data dictionary manager"
) #dictionary for " + which_name)
self.datadict_model = DataDict_Model(self.parent_window, self,
datadict_filename, show_frame,
"Record file data dictionary",
**kw_datadict)
self.datadict_model.display_view(self.bigcanvas.bigframe)
return True
'''
def load_datadict(self, datadict_filename="", mem_or_rec="rec"):
print("In main.load_datadict() with mem_or_rec=%s and datadict_filename=%s" % (str(mem_or_rec), str(datadict_filename) ) )
if which.lower()=="rec":
which_name = "Record File"
elif which.lower()=="mem":
which_name = "Memory File"
else:
self.error_message = "Invalid data dictionary type: " + which
print(self.error_message)
self.bigcanvas.bigframe.clear_canvas() #Unload (hide) all frame objects
self.load_splash("BigMatch data dictionary manager") #dictionary for " + which_name)
self.datadictobj.display_view(self.bigcanvas.bigframe)
return True
def load_recfile_datadict(self, datadict_filename=""):
show_frame = False #Don't immediately display the view
#if not self.datadictobj_recfile:
# self.datadictobj_recfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Record file data dictionary")
if self.datadictobj_recfile:
self.datadictobj_recfile = None
kw_datadict = {"mem_or_rec":"rec"}
self.datadictobj_recfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Record file data dictionary", **kw_datadict)
#if datadict_filename: #If a filename was passed to this function, store it to the DataDict object's "datadict_filename" property.
# self.datadictobj_recfile.datadict_filename = datadict_filename
result = self.load_datadict("rec", datadict_filename)
return result
def load_memfile_datadict(self, datadict_filename=""):
show_frame = False #Don't immediately display the view
#if not self.datadictobj_memfile:
# self.datadictobj_memfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Memory file data dictionary")
if self.datadictobj_memfile:
self.datadictobj_memfile = None
kw_datadict = {"mem_or_rec":"mem"}
self.datadictobj_memfile = DataDict_Model(self.parent_window, self, datadict_filename, show_frame, "Memory file data dictionary", **kw_datadict)
#if datadict_filename: #If a filename was passed to this function, store it to the DataDict object's "datadict_filename" property.
# self.datadictobj_memfile.datadict_filename = datadict_filename
result = self.load_datadict("mem", datadict_filename)
return result
'''
def load_blocking_passes(self):
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash(
"Chapin Hall's BigMatch user interface -- define blocking passes")
self.blockingpass_model = BlockingPass_Model(self.parent_window, self)
self.blockingpass_model.display_views(self.bigcanvas.bigframe,
self.how_many_blk_passes)
'''def load_blocking_pass_from_parmfile(self):
parmfile = os.path.join('C:\Greg', 'code', 'bigmatch_utilities', 'BigMatchGui', 'parmf.txt')
parmfileobj = BigmatchParmfile(parmfile)
for parm in parmfileobj.parms:
print("PARMFILE PARM: blkpass: %s, row_index: %s, row_type: %s, parms in row: %s, parm_index: %s, parm_type: %s, parm_value: %s" % (parm["blocking_pass"], parm["row_index"], parm["row_type"], parm["parms_in_row"], parm["parm_index"], parm["parm_type"], parm["parm_value"] ) )
'''
def load_match_review(self):
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash(
"Chapin Hall's BigMatch user interface -- review match results")
self.matchreview_model = MatchReview_Model(self.parent_window, self)
#self.matchreview_model.display_view(self.bigcanvas.bigframe)
self.matchreview_model.display_views(self.bigcanvas.bigframe, 1)
def load_combine_reviewed_results(self):
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash(
"Chapin Hall's BigMatch user interface -- combine reviewed match results"
)
self.matchreview_model = MatchReview_Model(self.parent_window, self)
#self.matchreview_model.display_view(self.bigcanvas.bigframe)
self.matchreview_model.display_views(self.bigcanvas.bigframe, 1)
def load_error_display(self):
print("In main.load_error_display()")
self.bigcanvas.bigframe.clear_canvas(
) #Unload (hide) all frame objects
self.load_splash("BigMatch utilities - error report")
if not self.error_list:
self.error_list = ["Unspecified error (main)"]
self.error_ui = Error_UI_Model(self.parent_window, self,
self.error_list)
self.error_ui.display_views(self.bigcanvas.bigframe,
len(self.error_list))
print("\n\nBack in MAIN.PY after error was displayed. \n\n")
'''def load_convert_file(self, source_format="sas", output_format="text"):
print("\nIn MAIN, load_convert_file() with source %s and output %s" % (source_format, output_format) )
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash("Chapin Hall's BigMatch user interface -- convert files")
self.convertfile_model = ConvertFile_Model(self.parent_window, self, source_format, output_format)
self.convertfile_model.display_view(self.bigcanvas.bigframe)'''
def instantiate_convert_file(self, source_format=None, output_format=None):
print("\nIn MAIN, load_convert_file() with source %s and output %s" %
(source_format, output_format))
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash(
"Chapin Hall's BigMatch user interface -- convert files")
self.convertfile_model = ConvertFile_Model(self.parent_window, self,
source_format,
output_format)
self.convertfile_model.display_view(self.bigcanvas.bigframe)
def instantiate_datafile_rdbms_ui(self,
source_format="txt",
output_format="sqlite"):
print("\nIn MAIN, load datafile_to_rdbms_ui()")
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash(
"Chapin Hall's BigMatch user interface -- convert text file to database table"
)
self.convertfile_model = Datafile_to_RDBMS_UI_Model(
self.parent_window, self, source_format, output_format)
self.convertfile_model.display_view(self.bigcanvas.bigframe)
def instantiate_readexport_rdbms_ui(self,
db_platform="sqlite",
output_format="text"):
print("\nIn MAIN, load ReadExport_Rdbms_UI()")
self.bigcanvas.bigframe.clear_canvas() #Hide all frame objects
self.load_splash(
"Chapin Hall's BigMatch user interface -- read/export data from database"
)
self.db_readexport_model = RDBMS_Read_Export_UI_Model(
self.parent_window, self, db_platform, output_format)
self.db_readexport_model.display_view(self.bigcanvas.bigframe)
def display_sqlite_data(self):
self.instantiate_readexport_rdbms_ui("sqlite")
def load_convert_file_sas_to_text(self):
self.instantiate_convert_file("sas", "text")
#self.convertfile_model.set_source_format("sas")
#self.convertfile_model.set_output_format("text")
def load_convert_file_csv_to_text(self):
self.instantiate_convert_file("csv", "text")
def load_convert_file_txt_to_sqlite(self):
self.instantiate_datafile_rdbms_ui("txt", "sqlite")
def generate_parmfile(self, filename_with_path=''):
print("This function is not yet available.")
def clear_canvas(self):
'''clear_canvas() calls the function of the same name, within the child frame within this object's child scrolling frame.
This is used when the user switches between Data Dictionary and Blocking Pass screens, or similar switch.'''
self.bigcanvas.bigframe.clear_canvas()
def refresh_main_canvas(self):
self.bigcanvas.bigframe.update_idletasks()
self.bigcanvas.update_idletasks()
self.bigcanvas.config(scrollregion=self.bigcanvas.bbox(
ALL)) #Canvas object needs to encompass new items
def set_startup_module_to_blocking_pass(self):
self.user.write_setting_to_config_file('cmd_onload',
'load_blocking_passes')
def set_startup_module_to_match_review(self):
self.user.write_setting_to_config_file('cmd_onload',
'load_match_review')
def set_startup_module_to_file_convert(self):
self.user.write_setting_to_config_file(
'cmd_onload', 'load_convert_file_csv_to_text')
def set_startup_module_to_data_dict(self):
self.user.write_setting_to_config_file('cmd_onload', 'load_datadict')
def startclean_new_datadict(self):
self.clear_canvas()
self.redisplay_module_after_error("datadict")
def startclean_new_blockingpass(self):
self.clear_canvas()
self.redisplay_module_after_error("blockingpass")
def redisplay_module_after_error(self, module_name="errordisplay"):
'''Note: This function might be executed after an error, but it might be initiated by the user if they want to wipe out what they started, and start fresh. '''
print("\n Back in main.py after error, calling module %s" %
(module_name))
if str(module_name).lower().strip() == "datadict":
self.load_datadict()
elif str(module_name).lower().strip() == "blockingpass":
self.load_blocking_passes()
elif str(module_name).lower().strip() == "matchreview":
self.load_match_review()
elif str(module_name).lower().strip() == "convertfile":
self.load_convert_file_csv_to_text()
elif str(module_name).lower().strip() == "errordisplay":
self.load_error_display()
else:
self.load_error_display()
print("\nEnd of Main.redisplay_module_after_error()")
def kill_module(self, module_name):
'''Set to nothing the object that is to be destroyed. This assumes that this BigMatch Controller holds the only reference to that object (which is most often the case)'''
print("\n Main.py kill_module(), killing module %s" % (module_name))
if str(module_name).lower().strip() == "datadict":
self.datadict_model = None
elif str(module_name).lower().strip() == "blockingpass":
self.blockingpass_model = None
elif str(module_name).lower().strip() == "matchreview":
self.matchreview_model = None
elif str(module_name).lower().strip() == "convertfile":
self.convertfile_model = None
print("\nEnd of Main.kill_module()")
def get_bigmatch_exe_location(self):
if self.host_name.lower() == "pennhurst1.chapinhall.org":
loc = os.path.join("/usr", "local", "bin", "bigmatch")
self.bigmatch_exe_location = loc
elif self.host_name.lower() == "chrsd1-gsanders":
loc = os.path.join("C:\Greg", "ChapinHall", "RecordLinking",
"BigMatch", "bigmatchqst.exe")
self.bigmatch_exe_location = loc
else:
self.bigmatch_exe_location = "bigmatch"
return self.bigmatch_exe_location
def call_bigmatch(self):
#subprocess.call(args, *, stdin=None, stdout=None, stderr=None, shell=False)
process = subprocess.Popen(self.bigmatch_exe_location,
stdout=subprocess.PIPE,
creationflags=0x08000000)
process.wait()
def get_command_for_file_convert_sas_to_text(self):
cmd_string = "self.load_convert_file('sas', 'text')"
print("\nReturning the string %s" % (cmd_string))
return cmd_string
def handle_error(self,
error_message='Error. Procedure cancelled.',
excp_type="",
excp_value="",
excp_traceback="",
continue_after_error=False,
abort_all=False,
calling_object_name=None,
module_to_reload_afterwards="errordisplay"):
print(
"\nTop of Main.handle_error(), error_message: %s, excp_type: %s, excp_value: %s, continue_after: %s"
% (error_message, excp_type, excp_value, continue_after_error))
self.error_message = error_message
self.add_error_to_list(self.error_message, excp_type, excp_value,
excp_traceback)
if self.logobject:
self.logobject.logit(
"\nCalling Common.handle_error with message '%s'" %
(self.error_message), True,
True) #Log this error in disk log and/or database
self.common.handle_error(
self.error_message, excp_type, excp_value, excp_traceback,
continue_after_error, abort_all
) #To make the error handler more generic, place it in a lightweight common functions lib so that it can be called even when this BigMatch controller is not in use.
if abort_all:
sys.exit(1) #Shut down the Python interpreter
if not continue_after_error:
if calling_object_name: #Whatever class/object called this error handler is unceremoniously dumped as a result
self.kill_module(calling_object_name)
self.clear_canvas()
self.redisplay_module_after_error(module_to_reload_afterwards)
print("\n\nEnd of Main.handle_error()")
def add_error_to_list(self,
error_message="Unspecified error (in main)",
excp_type="",
excp_value="",
excp_traceback=""):
errdict = {
"error_message": error_message,
"exc_type": excp_type,
"exc_value": excp_value,
"exc_traceback": excp_traceback
}
self.error_list.append(errdict)
return len(self.error_list)
class CommonRL():
debug = False
error_message = None
parent_window = None #Parent_window is the TKinter object itself (often known as "root"
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
user_response = None
datetime_str = None
def __init__(self, parent_window, controller):
self.parent_window = parent_window #Parent_wiondow is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\nIn CommonRL._init_", True, True )
def handle_error(self, error_message="Unspecified Error", excp_type="", excp_value="", excp_traceback=None, continue_after_error=False, abort_all=False):
print("\nIn CommonRL.handle_error(), message is '%s' abort_all='%s', continue_after_error='%s'. Type of logobject is %s" % (error_message, abort_all, continue_after_error, type(self.logobject)) )
self.error_message = error_message
#Date/time info
now = datetime.datetime.now()
self.datetime_str = str(now.year) + "-" + str(now.month) + "-" + str(now.day) + " " + str(now.hour) + ":" + str(now.minute) #+ str(now.second)
if self.logobject:
to_browser=False
to_console = to_logfile = to_db = message_is_error = True
print("\nIn CommonRL.handle_error(), about to call logobject with message '%s', excp_type: %s, excp_value: %s, type of traceback is %s" % (error_message, excp_type, excp_value, type(excp_traceback)) )
self.logobject.logit("\n##############################" + "\nERROR " + str(self.datetime_str), to_console, to_logfile, to_db, to_browser, message_is_error)
self.logobject.logit(str(error_message), to_console, to_logfile, to_db, to_browser, message_is_error) #Log this error in disk log and/or database
self.logobject.logit(str(excp_type), to_console, to_logfile, to_db, to_browser, message_is_error) #Log this error in disk log and/or database
self.logobject.logit(str(excp_value), to_console, to_logfile, to_db, to_browser, message_is_error) #Log this error in disk log and/or database
typ = str(type(excp_traceback)).lower().replace("<class '", "").replace("'>", "")
if typ == "traceback":
self.logobject.logit("Error Line: " + str(excp_traceback.tb_lineno), to_console, to_logfile, to_db, to_browser, message_is_error) #Log this error in disk log and/or database
title = "Warning"
if abort_all:
title = "Error"
elif continue_after_error:
title = "Warning"
elif not continue_after_error:
title = "Error"
else:
title = "Warning"
if not error_message:
error_message = "Unspecified error"
print("\nAbout to pop up error message.")
if str(error_message).strip()[-1] != ".":
error_message = error_message.strip() + "."
#**************************************************************************************************
#Display a TKinter error alert dialog box
text = error_message
if abort_all or not continue_after_error:
error_message += " Procedure cancelled."
user_response = tkinter.messagebox.showwarning(title, error_message)
#**************************************************************************************************
print("\nEnd of CommonRL.handle_error()")
if abort_all:
raise
sys.exit(1)
else:
if continue_after_error:
pass
else:
pass
return
class RDBMS_Read_Export_UI_Model():
debug = False
error_message = None
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
#sql_commands = [] #List of commands to create a new RDBMS table to hold the data from self.datafile
sql_text = None #SQL SELECT command entered by the user in the text box
sql_result = [] #List of lists containing the data retrieved - each item in the list is a list of column values.
sql_result_with_column_info = [] #List of lists of lists, each inner list containing column index value type and value. (Will this be useful in some instances?)
sql_result_as_single_string = [] #List of lists containing the data retrieved - each item in the list is a list of column values.
datafile = None #Data file path and filename
datafile_type = None #CSV or flat file?
datafile_handle = None #File Handle object for Python to access the data file (flat text)
csvfile_handle = None #File Handle object for Python to access CSV file
datafile_rowcount = 0 #Rows found in the data file
datafile_path_as_list = [] #Store the path and filename of the data file, split out into segments
datafile_pathonly = None #Path without the file name
datafile_nameonly = None #File name without path
#file_rows = None #Python list to hold the data file contents while traversing the file
row_index = 1 #Counter for iterating thru the data file
row_index_cumulative = 1 #Counter for rows inserted during the LIFE of this class instance (not just this iteration of import_data()).
db_conn = None #RDBMS connection object for storing data values
db_conn_validate = None #RDBMS connection object used for validating values
db_platform = None #RDBMS platform to be used for importing the flat file data (e.g., "sqlite3", "mysql")
db_catalog = None #RDBMS database (catalog) name where data will be saved
db_tablename = None #RDBMS table name
db_pathonly = None #DB file Path without the file name (Sqlite only)
#db_table_trunc = None #Prefix for RDBMS table name
#db_table_suffix = "" #Suffix for RDBMS table name - OPTIONAL, can be set to whatever is useful
db_catalog_for_validation = None #RDBMS database (catalog) name used for validation (only when specified--this is optional)
validation_by_row_or_column = "" #If "column" then invalid values are set to blank string and included in the save; validation by "row" is executed once per row, and false result means do not write this row to the result file.
row_validation_function = None #If validation_by_row_or_column = "row", execute this function to validate the row of data.
validation_parm_values = [] #The validation function(s) can retrieve these parameters, which can be populated by the data-write function.
colwidth_current = 0 #Temporary field used when parsing the data file - width of the current column
startpos_before_current = 0 #Temporary field used when parsing the data file - starting position of the current column, before its own width is added to the total positions already processed.
startpos_after_current = 0 #Temporary field used when parsing the data file
output_format = None #Sqlite3 by default
output_file = None #Sqlite file (if sqlite is the RDBMS)
datadict_file = None
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
filepathobj_datadict = None #FilePath object to allow user to select a data dictionary (for parsing text into distinct columns)
datadict = [] #List of columns read in from the data dictionary (if one was specified by the user)
controls = []
control_index = 0
converter = None #Object that handles the conversion
#conversion_func_name = None
csv_column_headers = [] #List of column headers in the data dictionary CSV
title = '' #Title to be displayed in the Frame object
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
btnLoadFile = None
btnSaveToFile = None
def __init__(self, parent_window, controller, db_platform=None, output_format=None, output_file=None, show_view=None, title='File converson', **kw): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
self.parent_window = parent_window #parent_window is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
now = datetime.datetime.now()
self.init_time = str(now.year) + str(now.month).rjust(2,'0') + str(now.day).rjust(2,'0') + ' ' + str(now.hour).rjust(2,'0') + ':' + str(now.minute).rjust(2,'0')
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\n\n____________________________________________________________________", True, True )
self.logobject.logit("\nIn RDBMS_Read_Export_UI_Model._init_: db_platform='%s'output_format='%s' -- and type(controller)=%s" % (db_platform, output_format, type(controller)), True, True )
self.db_platform = db_platform
self.output_format = output_format
self.output_file = output_file
self.title = title
self.show_view = show_view
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor=gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width=gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height=gl_frame_height
def set_output_format(self, output_format):
self.output_format = output_format
print("\nSetting the output_format to: %s" % (output_format) )
def display_data(self, db_platform=None, db_catalog=None):
if db_platform is None:
db_platform = self.db_platform
db_platform = str(db_platform).lower().strip()
print("\nTop of function display_data() - db_platform=%s" % (db_platform) )
#Read the DB records into memory
if db_platform[:6] == "sqlite":
self.read_sqlite_data()
#Display the DB records on screen (one batch at a time)
if not self.error_message:
self.display_view()
#Write the data to a file if a file was specified
if not self.error_message:
if self.output_file:
print("\n\nWriting data to text file - please wait...")
self.write_data_to_flat_file()
def read_sqlite_data(self):
#try:
if True:
if self.db_catalog is None:
self.error_message = "No sqlite database file was specified"
sqlcmd = self.sql_text.get()
if self.sql_text is None or not self.sql_text.get():
self.error_message = "No SQL statement was specified"
if sqlcmd.upper().strip()[:6] != "SELECT":
self.error_message = "Only SELECT commands can be executed."
if self.error_message:
print("\n\nERROR: %s" % (self.error_message) )
return
if self.debug: print("\nAbout to call open_db_connection()")
self.open_db_connection()
if self.debug: print("\n Type of db_conn: %s" % (str(type(self.db_conn)) ) )
if str(type(self.db_conn)) == "NoneType":
self.error_message = "Failed to open database connection for : '%s'" % (self.db_platform)
#Test - display tables in database #self.drop_db_table("ccts_root_20140402")
self.list_db_tables()
cursor = self.db_conn.cursor()
print("\n.....................................\nSQL: %s" % (self.sql_text.get()) )
cursor.execute(self.sql_text.get())
#COPY THE CURSOR RESULTS INTO AN ARRAY (list of lists of dicts)
self.sql_result_simple = "" #Quick read of the first row, just to verify that data was retrieved and view it on screen
ix = 0
print("\nResults:")
for row in cursor:
tuple_as_list = list(row)
tuple_as_list.insert(0, ix)
print("\nTuple_as_List: %s" % (tuple_as_list) )
self.sql_result.append(tuple_as_list) #Store the data to a list of lists
#############
#ALTERNATELY: Store the data and include column-level info (will this be useful in some cases?
row_as_list = [] #row_as_list will hold a row number and multiple COLUMN dictionaries {colnum, colvalue}
print("Row type: %s, len: %s, val: %s" % ( str(type(row)), len(row), row ) )
c = 0
for col in row: #ROW in a cursor is a tuple
if col is None:
#print("col: Type %s, val %s" % ( str(type(col)), col ) )
col = "None"
else:
#print("col: Type %s, val %s" % ( str(type(col)), col ) )
pass
typ = str(type(col)).lower().replace("<class '", "").replace("'>", "") #Data type of this value
if self.datadict_file:
width = self.get_width_for_column_pos(c)
if width:
col = str(col)[:width].ljust(width)
collist = [c, typ, col]
row_as_list.append(collist) #Add this column to the current row of "row_as_list" list
c += 1
self.sql_result_with_column_info.append(row_as_list)
#print("%s) %s %s %s %s %s" % (ix, row[0], row[1], row[2], row[3], row[4], ) )
'''if ix == 0: #Top row
self.sql_result_simple = str(row[0]) + " " + str(row[1]) #Display a bit of text to verify that we actually retrieved some data.
self.result_box.delete(0, END)
self.result_box.insert(END, self.sql_result_simple)'''
ix +=1
self.close_db_connection()
#Display the data that was stored in array (copy of cursor)
print("\n____________________________________")
r = 0
for row_as_list in self.sql_result_with_column_info:
print("\nARRAY ROW %s type %s... %s" % (row_as_list[0], type(row_as_list), row_as_list)) #Row number is item[0]
rowtext = ""
c = 0
for col in row_as_list:
print(" %s" % ( col ) ) #COL is a list containing the column number and the column value
rowtext += str(col[2]) + " " #COL[2] is the column's value
c += 1
print(rowtext)
rownum_plus_rowtext = [r, rowtext]
self.sql_result_as_single_string.append(rownum_plus_rowtext)
r +=1
if self.error_message:
print("\n\n ERROR: %s" % (self.error_message) )
return self.error_message
#except:
# self.error_message = "Error while reading database with %s catalog %s, statement: '%s' [%s %s %s]" % (self.db_platform, self.db_catalog, self.sql_text.get(), sys.exc_info()[0], sys.exc_info()[1], sys.exc_info()[2] )
# print("\n\nERROR: %s" % (self.error_message) )
def write_data_to_flat_file(self):
#*************************************
#Write the data values:
#*************************************
if not self.output_file:
self.error_message = "No output file was specified."
print("\n\nERROR: %s" % (self.error_message) )
return
self.logobject.logit("\n In RDBMS_Read_Export_UI_Model.write_data_to_flat_file()", True, True )
self.row_index = 0
curvalue = ""
print("\nLength of self.sql_result_as_single_string: %s" % (len(self.sql_result_as_single_string)) )
with open(self.output_file, 'w') as f:
ix = 0
for item in self.sql_result_as_single_string:
f.write("%s %s \n" % (item[1], ix) )
print("Row Value: %s %s" % (item[1], ix) )
ix += 1
f.close()
def open_db_connection(self):
#try:
if True:
if str(self.db_platform).lower().strip()[:6] == "sqlite":
if self.debug: print("\nAbout to open a Sqlite connection at: %s" % (self.db_catalog) )
self.db_conn = sqlite3.connect(self.db_catalog)
self.db_conn.text_factory = str
#except:
# self.error_message = "Error while opening database connection for %s catalog %s" % (self.db_platform, self.db_catalog) + " " + str(sys.exc_info()[0])
# print("\n\nERROR: %s" % (self.error_message) )
def close_db_connection(self):
if str(self.db_platform).lower().strip() == "sqlite3":
if self.db_conn is not None:
self.db_conn.close()
def check_db_connection_open(self, conn):
open = False
if str(self.db_platform).lower().strip() == "sqlite3":
try:
cmd = "select * from %s LIMIT 1" % (self.db_tablename)
conn.execute(cmd)
open = True
except:
pass
return open
def format_db_name(self, db_catalog="", copy_to_class_property=True):
if not db_catalog:
db_catalog = self.db_catalog #Default to whatever DB_CATALOG has been specified as a property of this object
if self.db_platform == "sqlite3":
#Sqlite3 uses files rather than an RDBMS application -- We want to specify the entire path, rather than just default to the Python path of the current directory.
if self.debug: print("\nDb_pathonly: %s, datafile_pathonly: %s" % (self.db_pathonly, self.datafile_pathonly) )
head, tail = os.path.split(db_catalog)
if self.debug: print("db_catalog head and tail: %s, %s" % (head, tail) )
if not head: #If the sqlite database is not specified as a .db filename with a full path
if self.debug: print("Db_catalog is not specified as an existing file.")
if self.db_pathonly:
if(db_catalog.lower().find(self.db_pathonly) == -1):
db_catalog = os.path.join(self.datafile_pathonly, db_catalog)
elif self.datafile_pathonly:
if(db_catalog.lower().find(self.datafile_pathonly.lower()) == -1):
db_catalog = os.path.join(self.datafile_pathonly, db_catalog)
if not db_catalog.lower()[-3:] == ".db":
db_catalog = db_catalog + ".db"
if self.debug: print("Sqlite DB file location (after formatting): %s" % (db_catalog) )
#Assume that this db_catalog should be stored as the db_catalog property of this object:
if copy_to_class_property:
self.db_catalog = db_catalog
return db_catalog
def list_db_tables(self):
#if self.db_conn is None:
self.open_db_connection()
with self.db_conn:
if self.db_platform == "sqlite3":
#cmd = "select * from sqlite_master;"
#for row in self.db_conn.execute(cmd):
# print("MASTER: %s %s %s %s %s" % ( row[0], row[1], row[2], row[3], row[4] ) ) #, row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9] ) )
cmd = "select name from sqlite_master where type = 'table';"
if self.debug: print("\nList DB Tables in db '%s':" % (self.db_catalog) )
for val in self.db_conn.execute(cmd):
if self.debug: print("TABLE: %s" % ( row[0] ) ) #, row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8], row[9] ) )
#self.close_db_connection()
#self.db_conn.close()
def instantiate_view_object(self, container):
self.view_object = RDBMS_Read_Export_UI_View(container, self)
return self.view_object
def copy_datadict_to_class_properties(self):
datadict = Datadict_Common() #Standard DataDict class
hdr_list = datadict.load_standard_datadict_headings(["bigmatch"]) #Make sure we are using the standard, updated list of column headings for the Data Dictionary
datadict = None #Erase the class instantiation when done to release memory
#Check our assumptions about which Data Dictionary column headings are in the standard:
if not "column_name" in hdr_list:
self.error_message = "Expected item 'column_name' to be in Data Dictionary header row"
if not "start_pos" in hdr_list:
self.error_message = "Expected item 'start_pos' to be in Data Dictionary header row"
if not "width" in hdr_list:
self.error_message = "Expected item 'width' to be in Data Dictionary header row"
if self.error_message:
print("\n \n ERROR: " + self.error_message)
try:
#Read the Data Dictionary and store information for each COLUMN as properties of the
self.logobject.logit("\nAdding columns to the dxr.DataDictionary", True, True)
with open(self.datadict_file, 'r') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',') #, quotechar='')
ix = 0
for row in csvreader:
if ix == 0: #Top row of CSV
self.build_column_header_list(row)
break
#Ascertain the correct position within the Data Dictionary where column name, width and startpos are stored
pos_colname = int(self.get_position_of_datadict_column("column_name"))
pos_width = int(self.get_position_of_datadict_column("width"))
pos_startpos = int(self.get_position_of_datadict_column("start_pos"))
ix = 0
for row in csvreader:
#NOTE: CSV reader is a forward-only reader -- and it has already read row 0, the top row. So now the next row WILL BE ROW 1!
#Add this column from the data dictionary to the list of columns that will be created in the RDBMS
#self.logobject.logit("Dict row " + str(ix) + ": " + row[0] + " " +row[1] + " " +row[2] + " " +row[3] + " " +row[4] + " " +row[5], True, True )
self.add_column(ix, row[pos_colname], "char", row[pos_width], row[pos_startpos], "file")
ix += 1
csvfile.close()
print("\nDatadict:")
for col in self.datadict:
print(col)
except:
self.controller.common.handle_error(self.error_message, False, False, "datafile_rdbms_ui")
def get_position_of_datadict_column(self, which_column):
which_column = which_column.lower().strip()
returnval = ""
for hdr in self.csv_column_headers:
self.logobject.logit("Seeking datadict column %s... Found: %s" % (which_column, hdr["col_hdr"].lower().strip() ), True, True )
if hdr["col_hdr"].lower().strip() == which_column:
returnval = hdr["col_index"]
break
self.logobject.logit("Seeking datadict column '%s'... Found position: %s" % (which_column, returnval), True, True )
return returnval
def add_column(self, index, name, type, width, startpos=None, datasource="file", source_func=None, validate=False, validate_func = None):
'''Add a column to the data dictionary. Data source is normally "file" because we read the value for this column from the specified positions in the current row of the data file. But if datasource="function" then get the value by calling the specified function.'''
#Column Name:
name = self.clean(name)
#Data source is normally "file" because we read the value for this column from the specified positions in the current row of the data file. But if datasource="function" then get the value by calling the specified function.
datasource = self.clean(datasource).lower().strip()
#Optionally, a value can be validated by submitting it to a specified function:
validate = self.clean(validate).lower().strip()
#Column Type, formatted for the specified database platform (MySql, Sqlite, etc.)
type_db_specific = self.get_datatype_for_dbplatform(self.clean(type))
try:
#Column width:
width = int(self.clean(width))
#Allow column width to be zero for CSV files only:
if width == 0:
self.error_message = "Column width may not be zero."
self.log_colwidth(width) #Function log_colwidth() allows us to calculate the starting position of each column based on the widths of all previous columns.
if startpos is None:
startpos = self.startpos_before_current #This value is calculated by function log_colwidth() -- this allows us to calculate the starting position of each column based on the widths of all previous columns.
#print("Adding column to datadict array: %s, %s, %s" % (name, width, startpos) )
except ValueError:
self.error_message = "Invalid column name, type or width: name='%s', type='%s', width=%s" % (name, type, width)
if self.error_message:
print("\n\n ERROR: %s" % (self.error_message) )
return self.error_message
#Add this column to the DataDict:
self.datadict.append({"index":index, "name":name, "width":width , "type":type_db_specific, "startpos":startpos, "datasource":datasource, "source_func":source_func, "validate":validate, "validate_func":validate_func })
return self.error_message
def get_datatype_for_dbplatform(self, data_type_generic):
data_type_generic = str(data_type_generic).lower().strip()
return_value = data_type_generic #By default, if no other type code is found for the specified type, return it unchanged (as opposed to returning a blank value or raising an error)
if self.db_platform == "sqlite3":
if data_type_generic == "string" or data_type_generic == "text" or data_type_generic == "char":
return_value = "text"
elif data_type_generic == "int" or data_type_generic == "integer" or data_type_generic == "numeric":
return_value = "int"
elif self.db_platform == "mysql":
if data_type_generic == "string" or data_type_generic == "text" or data_type_generic == "char":
return_value = "char"
elif data_type_generic == "int" or data_type_generic == "integer" or data_type_generic == "numeric":
return_value = "int"
return return_value
def clean(self, string_parm):
return_value = str(string_parm).replace(";", "")
return_value = str(string_parm).replace("@", "")
return_value = str(string_parm).replace("-", "_")
return return_value
def log_colwidth(self, colwidth):
self.colwidth_current = colwidth
self.startpos_before_current = self.startpos_after_current
#self.startpos_after_current = self.startpos_before_current + colwidth
self.startpos_after_current += colwidth
#print("self.startpos before and after current field (cur width=%s): %s, %s" % (colwidth, self.startpos_before_current, self.startpos_after_current) )
return colwidth
def build_column_header_list(self, hdr_row):
col_index = 0
for col_hdr in hdr_row:
col_hdr = col_hdr.strip()
self.logobject.logit("(%s) col_hdr: %s" % (col_index, col_hdr), True, True )
temp = {"col_index":col_index, "col_hdr":col_hdr, "max_width":0, "start_pos":0, "data_type":"", "selected":""}
self.csv_column_headers.append(temp)
col_index += 1
def get_width_for_column_pos(self, col_pos):
width = None
for col in self.datadict:
if col["index"] == col_pos:
width = col["width"]
break
print("Width of column in pos %s: %s" % (col_pos, width) )
return width
def display_view(self, container=None):
#Most often, we will display the form in the main class's BigCanvas.BigFrame "container".
self.logobject.logit("In DataFile_RDBMS_UI.Display_View, Type of Container is '%s'" % (str(type(container))), True, True )
kw_db = {"width":120, "background":self.bgcolor}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(container) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container, self.db_catalog)
self.display_user_buttons(container)
#print("\n In RDBMS_Read_Export_UI_Model.display_view(), about to call view_object.initUI().")
#Display the VIEW for this data dictionary
#self.view_object.initUI(**kw_db) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
print("\n In RDBMS_ReadExportUI_Model.display_view(), calling new_view_object.initUI().")
self.view_object.initUI(**kw_db) #DISPLAY THE DATA
self.controller.refresh_main_canvas()
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(".tk") == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(self.button_frame, "RDBMS_ReadExport_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(width=self.frame_width, background=self.bgcolor)
#Text entry box for user to define their SQL SELECT statement
lbl = Label(self.button_frame, text="SQL SELECT: ")
lbl.config(width=20, background=self.bgcolor, font=("Arial", 10, "bold"), borderwidth=0, padx=3, pady=3)
lbl.grid(row=0, column=1, sticky=W)
var = StringVar()
self.sql_text = var
self.sql_entry_box = Entry(self.button_frame, textvariable=var)
self.sql_entry_box.config(width=100, state=NORMAL, background='snow', borderwidth=2)
self.sql_entry_box.grid(row=0, column=2, sticky=W)
self.sql_entry_box.bind(sequence="<Enter>", func=self.handle_sqltext_event)
self.sql_entry_box.bind(sequence="<Button-1>", func=self.handle_sqltext_event)
self.sql_entry_box.bind(sequence="<FocusIn>", func=self.handle_sqltext_event)
self.sql_entry_box.bind(sequence="<FocusOut>", func=self.handle_sqltext_event)
#Spacer
lblblank = Label(self.button_frame, text=" ")
lblblank.config(width=8, background=self.bgcolor, font=("Arial", 10, "bold"), borderwidth=0, padx=3, pady=3)
lblblank.grid(row=0, column=3, sticky=W)
#Button to launch the conversion process:
self.btnSaveToFile = Button(self.button_frame, text="Display data", width=20, command=self.display_data)
self.btnSaveToFile.grid(row=0, column=4, sticky=W)
self.btnSaveToFile.config(state=DISABLED) #Do not enable this button unless the user has selected a data source
self.btnSaveToFile.config(padx=4, pady=4)
#Box for temp RESULT display
'''lbl = Label(self.button_frame, text="Result: ")
lbl.config(width=8, background=self.bgcolor, borderwidth=0, padx=3, pady=3)
lbl.grid(row=0, column=4, sticky=W)
var2 = StringVar()
self.result = var2
self.result_box = Entry(self.button_frame, textvariable=var2)
self.result_box.config(width=20, state=NORMAL, background='snow', borderwidth=2)
self.result_box.grid(row=0, column=5, sticky=W)'''
def handle_sqltext_event(self, parm=None):
print("\n SQL textbox stringvar: %s, stringvar value: %s ... %s '%s'" % (self.sql_text, self.sql_text.get(), parm, parm.widget) )
self.enable_disable_buttons()
def enable_disable_buttons(self):
if self.db_catalog:
if self.sql_text is not None and self.sql_text.get():
print("\nENABLING SAVE BUTTON")
self.btnSaveToFile.config(state=NORMAL)
else:
self.btnSaveToFile.config(state=DISABLED)
else:
self.btnSaveToFile.config(state=DISABLED)
def display_openfile_dialogs(self, container, default_filepath=''):
file_types = [('All files', '*'), ('DB files', '*.db;')]
kw_fpath = {"bgcolor":self.bgcolor, "frame_width":"", "frame_height":"", "file_category":"datadict"}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(self.parent_window, self, self.controller, "Sqlite .DB file:", open_or_save_as, "DbCatalogToLoad", file_types, **kw_fpath)
self.filepathobj_load_from.display_view(container) #Display the dialog for user to select a data dict file
file_types = [('All files', '*'), ('DB files', '*.db;')]
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(self.parent_window, self, self.controller, "File to save as (if exporting):", open_or_save_as, "OutputSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(container) #Display the dialog for user to Save As... a new data dict file
open_or_save_as = "open"
self.filepathobj_datadict = FilePath_Model(self.parent_window, self, self.controller, "Data dictionary (if exporting):", open_or_save_as, "DatadictToLoad", file_types, **kw_fpath)
self.filepathobj_datadict.display_view(container) #Display the Open File dialog
def update_filepath(self, file_name_with_path='', callback_string='', alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
#self.logobject.logit("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string), True, True )
print("Master RDBMS_Read_Export_UI_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string))
if callback_string.lower().strip()[:4] == "load" or callback_string.lower().strip()[:4] == "open":
if str(callback_string).lower()[4:].find("datadict") != -1: #User selected a DataDictionary file
self.logobject.logit("datadict_file is being set to %s" % (file_name_with_path), True, True )
self.datadict_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.db_catalog because of the "callback string" returned by the FilePath object.
if self.datadict_file: #Refresh the view when the user selects a new file.
self.copy_datadict_to_class_properties() #Copy the data dictionary into a globally accessible list
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_grid() #Remove all existing values from the grid
else: #User selected the DB Source Catalog
self.db_catalog = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.db_catalog because of the "callback string" returned by the FilePath object.
if self.db_catalog: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_grid() #Remove all existing values from the grid
elif callback_string.lower().strip()[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.output_file = file_name_with_path
self.enable_disable_buttons()
self.update_master_paths(file_name_with_path)
self.update_initial_dir_for_file_open_dialogs()
def update_master_paths(self, file_name_with_path):
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
print("\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s" % (self.controller.dir_last_opened, self.controller.datadict_dir_last_opened, self.controller.rec_datadict_last_opened, self.controller.mem_datadict_last_opened) )
def update_initial_dir_for_file_open_dialogs(self):
'''In addition to tracking "last file opened" at the main controller level, we also want to notify every FilePath object when the user has opened a new file, so that they can adjust thir Initial DIr properties to the location just opened.'''
self.filepathobj_load_from.calc_initial_dir_for_file_open(self.db_catalog, "datadict", "")
self.filepathobj_save_to.calc_initial_dir_for_file_open(self.output_file, "datadict", "")
self.filepathobj_datadict.calc_initial_dir_for_file_open(self.datadict_file, "datadict", "")
def add_control_to_list(self, object, var, **kw):
control = Control(self, object, var, self.control_index, **kw)
self.controls.append(control) #self.controls is a list of the screen controls, which can be traversed after user has finished.
self.control_index += 1
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
class CommonRL():
debug = False
error_message = None
parent_window = None #Parent_window is the TKinter object itself (often known as "root"
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
user_response = None
datetime_str = None
def __init__(self, parent_window, controller):
self.parent_window = parent_window #Parent_wiondow is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\nIn CommonRL._init_", True, True)
def handle_error(self,
error_message="Unspecified Error",
excp_type="",
excp_value="",
excp_traceback=None,
continue_after_error=False,
abort_all=False):
print(
"\nIn CommonRL.handle_error(), message is '%s' abort_all='%s', continue_after_error='%s'. Type of logobject is %s"
% (error_message, abort_all, continue_after_error,
type(self.logobject)))
self.error_message = error_message
#Date/time info
now = datetime.datetime.now()
self.datetime_str = str(now.year) + "-" + str(
now.month) + "-" + str(now.day) + " " + str(now.hour) + ":" + str(
now.minute) #+ str(now.second)
if self.logobject:
to_browser = False
to_console = to_logfile = to_db = message_is_error = True
print(
"\nIn CommonRL.handle_error(), about to call logobject with message '%s', excp_type: %s, excp_value: %s, type of traceback is %s"
% (error_message, excp_type, excp_value, type(excp_traceback)))
self.logobject.logit(
"\n##############################" + "\nERROR " +
str(self.datetime_str), to_console, to_logfile, to_db,
to_browser, message_is_error)
self.logobject.logit(
str(error_message), to_console, to_logfile, to_db, to_browser,
message_is_error) #Log this error in disk log and/or database
self.logobject.logit(
str(excp_type), to_console, to_logfile, to_db, to_browser,
message_is_error) #Log this error in disk log and/or database
self.logobject.logit(
str(excp_value), to_console, to_logfile, to_db, to_browser,
message_is_error) #Log this error in disk log and/or database
typ = str(type(excp_traceback)).lower().replace("<class '",
"").replace(
"'>", "")
if typ == "traceback":
self.logobject.logit(
"Error Line: " + str(excp_traceback.tb_lineno), to_console,
to_logfile, to_db, to_browser, message_is_error
) #Log this error in disk log and/or database
title = "Warning"
if abort_all:
title = "Error"
elif continue_after_error:
title = "Warning"
elif not continue_after_error:
title = "Error"
else:
title = "Warning"
if not error_message:
error_message = "Unspecified error"
print("\nAbout to pop up error message.")
if str(error_message).strip()[-1] != ".":
error_message = error_message.strip() + "."
#**************************************************************************************************
#Display a TKinter error alert dialog box
text = error_message
if abort_all or not continue_after_error:
error_message += " Procedure cancelled."
user_response = tkinter.messagebox.showwarning(title, error_message)
#**************************************************************************************************
print("\nEnd of CommonRL.handle_error()")
if abort_all:
raise
sys.exit(1)
else:
if continue_after_error:
pass
else:
pass
return
class Datafile_to_RDBMS_UI_Model():
debug = False
error_message = None
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
dxr = None #Instance of the Datafile_to_RDBMS() class
source_format = None #Flat text file by default
output_format = None #Sqlite3 by default
source_file = None #Name and path of the source file to be converted
output_file = None #Sqlite file (if sqlite is the RDBMS)
datadict_file = None
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
filepathobj_datadict = None #FilePath object to allow user to select a data dictionary (for parsing text into distinct columns)
converter = None #Object that handles the conversion
#conversion_func_name = None
csv_column_headers = [] #List of column headers in the data dictionary CSV
title = '' #Title to be displayed in the Frame object
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
btnLoadFile = None
btnSaveToFile = None
def __init__(self, parent_window, controller, source_format=None, output_format=None, source_file=None, output_file=None, show_view=None, title='File converson', **kw): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
self.parent_window = parent_window #parent_window is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
now = datetime.datetime.now()
self.init_time = str(now.year) + str(now.month).rjust(2,'0') + str(now.day).rjust(2,'0') + ' ' + str(now.hour).rjust(2,'0') + ':' + str(now.minute).rjust(2,'0')
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\n\n____________________________________________________________________", True, True )
self.logobject.logit("\nIn Datafile_to_RDBMS_UI_Model._init_: source_file='%s' -- and type(controller)=%s" % (source_file, type(controller)), True, True )
self.source_format = source_format
self.output_format = output_format
self.source_file = source_file
self.output_file = output_file
self.title = title
self.show_view = show_view
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor=gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width=gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height=gl_frame_height
def set_source_format(self, source_format):
self.source_format = source_format
print("\nSetting the source_format to: %s" % (source_format) )
def set_output_format(self, output_format):
self.output_format = output_format
print("\nSetting the output_format to: %s" % (output_format) )
def convert_file(self, source_format=None, output_format=None):
'''Called after the user has selected a source file and an output file '''
if not source_format:
source_format = self.source_format
if not output_format:
output_format = self.output_format
source_format = source_format.lower().strip()
output_format = output_format.lower().strip()
#Based on source and output formats, call the appropriate function to handle the conversion.
if source_format == "txt" and output_format[:6] == "sqlite":
self.convert_txt_to_sqlite(self.source_file, self.output_file)
def convert_txt_to_sqlite(self, source_file=None, output_file=None):
'''DOCSTRING '''
if not source_file:
source_file = self.source_file
if not output_file:
output_file = self.output_file
#Instantiate the Datafile_to_RDBMS class
self.converter = Datafile_to_RDBMS()
self.logobject.logit("\nAbout to convert file '%s' to table in RDBMS '%s'" % (source_file, output_file), True, True )
self.converter.datafile = source_file
head, tail = os.path.split(source_file)
db_tablename = tail.lower().strip()
dotpos = db_tablename.find(".")
if(dotpos != -1):
db_tablename = db_tablename[:dotpos].replace(".", "")
self.converter.db_tablename = db_tablename
self.converter.db_catalog = output_file
self.converter.db_platform = "sqlite3"
self.logobject.logit("\n Type of self.converter: %s" % (str(type(self.converter))), True, True )
#if(str(type(self.converter)).lower().find("datafile_to_rdbms") == -1): #When the class is first instantiated, load the data dictionary
if len(self.converter.datadict) == 0:
self.copy_datadict_to_class_properties()
self.converter.display_datadict()
#*******************************************
self.converter.import_data()
#*******************************************
return self.converter.error_message
def instantiate_view_object(self, container):
self.view_object = Datafile_to_RDBMS_UI_View(container, self)
return self.view_object
def copy_datadict_to_class_properties(self):
datadict = Datadict_Common() #Standard DataDict class
hdr_list = datadict.load_standard_datadict_headings(["bigmatch"]) #Make sure we are using the standard, updated list of column headings for the Data Dictionary
datadict = None #Erase the class instantiation when done to release memory
#Check our assumptions about which Data Dictionary column headings are in the standard:
if not "column_name" in hdr_list:
self.error_message = "Expected item 'column_name' to be in Data Dictionary header row"
if not "start_pos" in hdr_list:
self.error_message = "Expected item 'start_pos' to be in Data Dictionary header row"
if not "width" in hdr_list:
self.error_message = "Expected item 'width' to be in Data Dictionary header row"
if self.error_message:
print("\n \n ERROR: " + self.error_message)
try:
#Read the Data Dictionary and store information for each COLUMN as properties of the
self.logobject.logit("\nAdding columns to the dxr.DataDictionary", True, True)
with open(self.datadict_file, 'r') as csvfile:
csvreader = csv.reader(csvfile, delimiter=',') #, quotechar='')
count = 0
for row in csvreader:
if count == 0: #Top row of CSV
self.build_column_header_list(row)
break
#Ascertain the correct position within the Data Dictionary where column name, width and startpos are stored
pos_colname = int(self.get_position_of_datadict_column("column_name"))
pos_width = int(self.get_position_of_datadict_column("width"))
pos_startpos = int(self.get_position_of_datadict_column("start_pos"))
if self.debug: print("\npos_colname=%s, pos_width=%s, pos_startpos=%s" % (pos_colname, pos_width, pos_startpos) )
count = 0
for row in csvreader:
#NOTE: CSV reader is a forward-only reader -- and it has already read row 0, the top row. So now the next row WILL BE ROW 1!
#Add this column from the data dictionary to the list of columns that will be created in the RDBMS
#Determine whether this is a blank row in the file
if self.row_is_empty(row):
continue
if self.debug: self.logobject.logit("Dict row " + str(count) + ": " + row[0] + ", " + row[1] + ", " + row[2], True, True, True )
self.converter.add_column(row[pos_colname], "char", row[pos_width], row[pos_startpos], "file")
count += 1
csvfile.close()
except:
self.controller.common.handle_error(self.error_message, False, False, "datafile_rdbms_ui")
if self.debug: raise
def get_position_of_datadict_column(self, which_column):
which_column = which_column.lower().strip()
returnval = ""
for hdr in self.csv_column_headers:
self.logobject.logit("Seeking datadict column %s... Found: %s" % (which_column, hdr["col_hdr"].lower().strip() ), True, True )
if hdr["col_hdr"].lower().strip() == which_column:
returnval = hdr["col_index"]
break
self.logobject.logit("Seeking datadict column '%s'... Found position: %s" % (which_column, returnval), True, True )
return returnval
def build_column_header_list(self, hdr_row):
col_index = 0
for col_hdr in hdr_row:
col_hdr = col_hdr.strip()
self.logobject.logit("(%s) col_hdr: %s" % (col_index, col_hdr), True, True )
temp = {"col_index":col_index, "col_hdr":col_hdr, "max_width":0, "start_pos":0, "data_type":"", "selected":""}
self.csv_column_headers.append(temp)
col_index += 1
def display_view(self, container=None):
#Most often, we will display the form in the main class's BigCanvas.BigFrame "container".
self.logobject.logit("In DataFile_RDBMS_UI.Display_View, Type of Container is '%s'" % (str(type(container))), True, True )
kw_dict = {"width":self.frame_width, "background":self.bgcolor, "borderwidth":2, "padx":3, "pady":3}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(container) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container, self.source_file)
self.display_user_buttons(container)
#print("\n In Datafile_to_RDBMS_UI_Model.display_view(), about to call view_object.initUI().")
#Display the VIEW for this data dictionary
#self.view_object.initUI(**kw_dict) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
#self.view_object.display_datadict_in_grid(**kw_dict) #When first initiated, the View object will call this during its InitUI() method. But to refresh it afterwards, call display_datadict_in_grid() explicitly.
self.controller.refresh_main_canvas()
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(".tk") == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(self.button_frame, "Datfile_to_RDBMS_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(background=self.bgcolor)
#Button to launch the conversion process:
self.btnSaveToFile = Button(self.button_frame, text="Convert File", width=30, command=self.convert_file) #convert_txt_to_sqlite
self.btnSaveToFile.grid(row=0, column=1, sticky=W)
self.btnSaveToFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to save as
def display_openfile_dialogs(self, container, default_filepath=''):
file_types = [('All files', '*'), ('SAS files', '*.sas7bdat; *.txt')]
kw_fpath = {"bgcolor":self.bgcolor, "frame_width":"", "frame_height":"", "file_category":"datadict"}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(self.parent_window, self, self.controller, "Source file to load:", open_or_save_as, "ConvertSourceToLoad", file_types, **kw_fpath)
self.filepathobj_load_from.display_view(container) #Display the dialog for user to select a data dict file
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(self.parent_window, self, self.controller, "Sqlite .DB file:", open_or_save_as, "ConvertOutputToSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(container) #Display the dialog for user to Save As... a new data dict file
open_or_save_as = "open"
self.filepathobj_datadict = FilePath_Model(self.parent_window, self, self.controller, "Data dictionary:", open_or_save_as, "DatadictToLoad", file_types, **kw_fpath)
self.filepathobj_datadict.display_view(container) #Display the Open File dialog
def update_filepath(self, file_name_with_path='', callback_string='', alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
#self.logobject.logit("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string), True, True )
print("Master Datafile_to_RDBMS_UI_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string))
if callback_string.lower().strip()[:4] == "load" or callback_string.lower().strip()[:4] == "open":
if str(callback_string).lower()[4:].find("datadict") != -1: #User selected a DataDictionary file
self.logobject.logit("datadict_file is being set to %s" % (file_name_with_path), True, True )
self.datadict_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.source_file because of the "callback string" returned by the FilePath object.
if self.datadict_file: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid() #Remove all existing values from the grid
else: #User selected the Source File
self.source_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.source_file because of the "callback string" returned by the FilePath object.
if self.source_file: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid() #Remove all existing values from the grid
elif callback_string.lower().strip()[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.output_file = file_name_with_path
if self.output_file:
self.btnSaveToFile.config(state=NORMAL)
else:
self.btnSaveToFile.config(state=DISABLED)
self.update_master_paths(file_name_with_path)
self.update_initial_dir_for_file_open_dialogs()
def update_master_paths(self, file_name_with_path):
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
print("\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s" % (self.controller.dir_last_opened, self.controller.datadict_dir_last_opened, self.controller.rec_datadict_last_opened, self.controller.mem_datadict_last_opened) )
def update_initial_dir_for_file_open_dialogs(self):
'''In addition to tracking "last file opened" at the main controller level, we also want to notify every FilePath object when the user has opened a new file, so that they can adjust thir Initial DIr properties to the location just opened.'''
self.filepathobj_load_from.calc_initial_dir_for_file_open(self.source_file, "datadict", "")
self.filepathobj_save_to.calc_initial_dir_for_file_open(self.output_file, "datadict", "")
self.filepathobj_datadict.calc_initial_dir_for_file_open(self.datadict_file, "datadict", "")
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
def row_is_empty(self, row):
isempty = False
rowcheck = ','.join(row)
rowcheck = rowcheck.replace(",", "").replace("'", "").replace("\n", "").replace("?", "").strip()
#print("Rowcheck: %s" % (rowcheck))
if rowcheck == "":
#print("ROW IS EMPTY. SKIP IT.")
isempty = True
return isempty
class FilePath_Model():
debug = False
error_message = None
parent_window = None
controller = None #Controller is the BigMatch MAIN class file
master = None #Master is the object needing to locate a file. This might be the DataDict class, the BlockingPass class or another object.
logobject = None #Instantiation of CHLog class
file_types = []
file_name = None
file_path = None
file_name_with_path = None
view_object = None
initial_dir = None #Initial dir is the default location where the user's navigation will start (example: "C:\Temp\"). This can be set and re-set by the object that created it.
title = ''
bgcolor = ''
frame_width = None
frame_height = None
show_view = None
open_or_save_as = 'open' #open_or_save_as specifies whether this FilePath dialog will open an existing file (tkinter "askopenfilename"), or prompt user to "Save As..." (tkinter "asksaveasfilename")
file_category = None #Examples: 'DataDict', 'ParmFile', 'MatchResult'
alias = '' #Many instances of this class might be visible at any given time, and it's difficult to tell which one is executing what code without some kind of label that can be output for debugging.
def __init__(self, parent_window, master, controller, title='', open_or_save_as='open', alias='', file_types=[('All files', '*'),('Text files', '*.csv;*.txt;*.dat')], **kw): #bgcolor=gl_frame_color, file_types=[('All files', '*'),('Text files', '*.csv;*.txt')], frame_width=gl_frame_width, frame_height=gl_frame_height
if parent_window:
self.parent_window = parent_window
if master:
self.master = master #Master is the object needing to locate a file. This might be the DataDict class, the BlockingPass class or another object.
enable_logging = False
if controller:
self.controller = controller #Controller is the BigMatchController class in main.py
enable_logging = self.controller.enable_logging
self.logobject = CHLog(enable_logging)
#if initial_dir: #Initial_Dir is now included in the **Kwargs
# self.initial_dir = str(initial_dir)
if title:
self.title = title #Initial_Dir is now in the **Kwargs
if open_or_save_as:
self.open_or_save_as = open_or_save_as #open_or_save_as specifies whether this FilePath dialog will open an existing file (tkinter "askopenfilename"), or prompt user to "Save As..." (tkinter "asksaveasfilename")
if alias:
self.alias = alias
if not self.file_types:
self.file_types = [('All files', '*'), ('Text files', '*.csv;*.txt;*.dat')]
#kwargs:
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor=gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width=gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height=gl_frame_height
if self.check_key_exists("initial_dir", **kw):
self.initial_dir = str(kw["initial_dir"])
if self.initial_dir is None:
if self.controller.dir_last_opened:
self.initial_dir = self.controller.dir_last_opened
else:
self.initial_dir = os.path.dirname(os.path.realpath(__file__)) #Do not leave initial_dir empty because it causes an error in the open_file dialogs
if self.check_key_exists("file_category", **kw):
self.file_category = kw["file_category"]
print("\n FILE TYPES FOR OPEN FILE DIALOG: (type is: %s" % (str(type(self.file_types)) ) )
print(self.file_types)
def update_filepath_display(self, file_name_with_path):
'''When the user selects a file, display the file name and path in the Tk Entry box '''
self.file_name_with_path = file_name_with_path
self.view_object.textbox.configure(state=NORMAL) #Entry widget's state must be NORMAL in order to write or delete. But we need it to stay in "readonly" mode at all other times. This is because if we allow the user to enter in a file name, this does not trigger the events that are triggered by the "Browse" button, and those events are crucial.
self.view_object.textbox.delete(0, END) #does not seem to work?
self.view_object.textbox.insert(END, self.file_name_with_path)
self.view_object.textbox.configure(state="readonly") #Entry widget's state must be NORMAL in order to write or delete. But we need it to stay in "readonly" mode at all other times. This is because if we allow the user to enter in a file name, this does not trigger the events that are triggered by the "Browse" button, and those events are crucial.
def locate_file(self, file_types=[('All files', '*'), ('Text files', '*.csv;*.txt')], notify_master=True, callback_string=''):
'''See display_user_buttons() below in View. The buttons call either locate_file() or save_as_file(), depending on whether the user is loading or saving to a file. '''
#file_types = [('All files', '*'), ('Data files', '*.dat'), ('Text files', '*.csv;*.txt')]
file_name_with_path = tkinter.filedialog.askopenfilename(parent=self.parent_window, filetypes=self.file_types, title='Choose a file', initialdir=self.initial_dir) #initialdir=self.current_dir
if file_name_with_path != '' and file_name_with_path is not None:
self.file_name_with_path = file_name_with_path
print("\n" + "In locate_file, file_name_with_path = '" + file_name_with_path + "'")
if self.view_object is None:
self.display_view()
self.update_filepath_display(file_name_with_path) #Refresh the textbox displaying the selected file
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head #Store the path last opened by the user and set this as a default starting point for FileOpen dialogs
if notify_master: #A FileOpen dialog normally serves the purposes of another class, such as Data Dict or BlockingPass. The Master class needs to know when the user selects a file.
print("Notifying the master object that the user has selected a new file: %s" % (file_name_with_path) )
if callback_string == '': #Callback string is crucial for assigning the opened file's name and location to the correct class property in the class that created this Open File dialog!
callback_string = "load, " + self.alias
self.master.update_filepath(file_name_with_path, callback_string, self.alias)
return file_name_with_path
def save_as_file(self, file_types=[('All files', '*'), ('Text files', '*.csv;*.txt')], notify_master=True, callback_string=''):
'''See display_user_buttons() below in View. The buttons call either locate_file() or save_as_file(), depending on whether the user is loading or saving to a file. '''
file_name_with_path = tkinter.filedialog.asksaveasfilename(parent=self.parent_window, filetypes=self.file_types, title='Choose a file', initialdir=self.initial_dir) #initialdir=self.current_dir
if file_name_with_path != '' and file_name_with_path is not None:
self.file_name_with_path = file_name_with_path
print("\n" + "In save_as_file, self.file_name_with_path = '" + self.file_name_with_path + "'")
if self.view_object is None:
self.display_view()
self.update_filepath_display(self.file_name_with_path) #Refresh the textbox displaying the selected file
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head #Store the path last opened by the user and set this as a default starting point for FileOpen dialogs
if notify_master: #A FileOpen dialog normally serves the purposes of another class, such as Data Dict or BlockingPass. The Master class needs to know when the user selects a file.
print("Notifying the master object that the user has selected a new file: %s" % (file_name_with_path) )
if callback_string == '':
callback_string = "save_as, " + self.alias #Callback string is crucial for assigning the opened file's name and location to the correct class property in the class that created this Open File dialog!
self.master.update_filepath(file_name_with_path, callback_string, self.alias) #Alert the parent module (datadict, blockingpass, etc.) that a file name/path was changed!
return file_name_with_path
def clear_file(self):
self.file_name_with_path = ""
self.update_filepath_display("") #Refresh the textbox displaying the selected file
callback_string = self.open_or_save_as
self.master.update_filepath("", callback_string) #Alert the parent module (datadict, blockingpass, etc.) that a file name/path was changed!
def calc_initial_dir_for_file_open(self, current_file='', file_category='', mem_or_rec=''):
initial_dir = None
if current_file:
if os.path.isfile(current_file): #If a file is specified as initial directory, use the FOLDER, not the file.
head, tail = os.path.split(current_file)
initial_dir = head
else:
file_category = file_category.lower().strip() #It's useful to know whether the file being opened is a DATADICT, PARM, etc. because the main BigMatch controller object (main.py) stores a record of the last folder location navigated to for each major file type.
if file_category == "datadict":
if mem_or_rec.lower() == "mem" and self.controller.mem_datadict_last_opened:
head, tail = os.path.split(self.controller.mem_datadict_last_opened)
initial_dir = head
elif mem_or_rec.lower() == "rec" and self.controller.rec_datadict_last_opened:
head, tail = os.path.split(self.controller.rec_datadict_last_opened)
initial_dir = head
else:
initial_dir = self.controller.datadict_dir_last_opened
elif file_category == "parmfile":
initial_dir = self.controller.parmfile_dir_last_opened
self.logobject.logit("CalcInitDir(), parmfile_dir_last_opened: %s" % (self.controller.parmfile_dir_last_opened), True, True )
if not initial_dir:
initial_dir = self.controller.dir_last_opened #If we can't find a more specific history of user's previous folders opened, use the more generic:
self.logobject.logit("CalcInitDir(), found no specific matches so setting to %s" % (self.controller.dir_last_opened), True, True )
if not initial_dir:
#if no other strategy is found, punt and use the current executing program's folder
initial_dir = os.path.dirname(os.path.realpath(__file__))
self.logobject.logit("In calc_initial_dir(), initdir is %s" % (initial_dir), True, True )
#Set the Initial_Dir property to the calculated value:
self.initial_dir = initial_dir
return initial_dir
def instantiate_view_object(self, container):
self.view_object = FilePath_View(container, self)
def display_view(self, container=None, open_or_save_as=None):
#open_or_save_as specifies whether this FilePath dialog will open an existing file (tkinter "askopenfilename"), or prompt user to "Save As..." (tkinter "asksaveasfilename")
if open_or_save_as:
self.open_or_save_as = open_or_save_as
else:
open_or_save_as = self.open_or_save_as
if open_or_save_as is None:
open_or_save_as = "open"
self.open_or_save_as = open_or_save_as
if container == None:
container = self.controller.bigcanvas.bigframe #default objects on which other widgets are displayed.
if self.view_object is None:
self.instantiate_view_object(container)
self.view_object.initUI(open_or_save_as, background=self.bgcolor, borderwidth=2, padx=0, pady=0) #DISPLAY THE FRAME OBJECT ON SCREEN
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
class DataDict_Model():
debug = False
error_message = None
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
datadict_file_to_load_from = None #Data dictionary file name and path, which will be loaded into the entry grid if user requests.
#datadict_loadfile_displaybox = None #A textbox where the data dict file to be loaded will be displayed
datadict_file_to_save_to = None #Data dictionary file name and path, to which the metadata will be saved if user requests.
#datadict_savefile_displaybox = None #A textbox where the save-to data dict file will be displayed
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
title = '' #Title to be displayed in the Frame object
mem_or_rec = None #Is this datadict associated with a Record File or a Memory File?
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
#Arrays to hold the contents of the Data Dictionary:
meta_columns = None #List of column names within the data dictionary CSV (column_name, column_width, etc.)
meta_rownums = None #For convenience, a simple List of integers that will be displayed vertically on the left side of the Data Dict grid display.
meta_values = None #List of values for each cell within the data dictionary CSV (actually a list of lists - a list of "rows" from the data dictionary CSV)
meta_values_after_edit = [] #List of values from the Entry Grid, AFTER the user has modified the values and clicked SAVE.
btnLoadDictFile = None
btnSaveToDictFile = None
def __init__(self, parent_window, controller, datadict_file_to_load_from=None, show_view=None, title='Data dictionary', **kw): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
#Frame.__init__(self, parent_window)
self.parent_window = parent_window #parent_wiondow is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
self.title = title
self.show_view = show_view
self.datadict_file_to_load_from = datadict_file_to_load_from
self.logobject = CHLog()
self.logobject.logit("\n" + "In DataDict._init_, datadict_file_to_load_from = '" + str(self.datadict_file_to_load_from) + "' -- and show_view=" + str(self.show_view), True, True )
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor=gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width=gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height=gl_frame_height
if self.check_key_exists("mem_or_rec", **kw):
self.mem_or_rec = str(kw["mem_or_rec"]).lower().strip()
def read_datadict_file_into_arrays(self, file_name):
'''A data dictionary CSV file is read into arrays (which are properties of this Model object). From the arrays they can be written back to a CSV file later. '''
self.logobject.logit("\n" + "In DataDict.read_datadict_file_into_arrays, datadict_file_to_load_from = '" + str(file_name) + "'", True, True)
self.meta_columns = []
self.meta_rownums = []
self.meta_values = []
datadict_arrays = {}
f = open(file_name, "r")
content = f.readlines()
i = 0
for line in content:
print(line)
if(i==0): #Header row in the CSV file
self.meta_columns = line.split(",")
else:
meta_temp = line.split(",") #meta_temp is a LIST consisting of one row from the data dictionary
self.meta_values.append(meta_temp) #meta_values is a LIST of LISTS, consisting of one "outer" list representing all the rows from the data dictionary, and an "inner" list consisting of the cell values for a single row.
self.meta_rownums.append(str(i))
i += 1
def write_datadict_from_entrygrid(self, file_name=None):
if file_name is None:
file_name = self.datadict_file_to_save_to
print("\n Top of write_datadict_from_entrygrid(), file_name is '%s'" % (file_name) )
#Call the View object to retrieve Entry Grid values (after they have been update by the user).
if file_name:
del self.meta_values_after_edit[:]
self.meta_values_after_edit = [] #Clear out any previous entries
#Load the entry grid items into the meta_values_after_edit list:
self.meta_values_after_edit = self.view_object.retrieve_grid_values()
if self.meta_values_after_edit:
newrow = []
self.logobject.logit("Dict file in write_datadict_from_entrygrid(): %s ...File will be deleted if it already exists." % ( str(file_name) ), True, True )
if os.path.isfile(file_name):
os.remove(file_name) #Should not need to delete the file, but it has been appending new text at end of file instead of erasing existing text and re-writing-- even though the file open uses "w" not "w+", and it is closed after every write session.
with open(file_name, 'w', encoding='UTF8') as csvfile:
csvwriter = csv.writer(csvfile, delimiter=',', quotechar='"', quoting=csv.QUOTE_MINIMAL, lineterminator='\n') #os.linesep) #chr(13) + chr(10)
#Get a list of column names retrieved from the Entry Grid -- these are CSV header columns.
j = 0;
for col in self.view_object.entry_grid.colList:
newcol = str(col).replace("\n", "").strip()
newcol = newcol.replace(chr(10), "")
newcol = newcol.replace(chr(13), "")
if (j == 0 and newcol == ''): #First "column name" is usually blank because it is the row-numbers column, which has no header caption
pass
else:
newrow.append(newcol)
j += 1
csvwriter.writerow(newrow) #Write the Column Headers to the CSV file
newrow = []
i = 0
for row in self.meta_values_after_edit:
self.logobject.logit("\n In write_datadict(), ROW: %s (length is %s and type is %s" % (str(i), str(len(row)), str(type(row)) ), True, True)
self.logobject.logit(row, True, True)
newrow = []
j = 0
for col in row:
#if col.find("\n") != -1:
# print("Reassigning col %s in row %s to a string value," % (str(j), str(i)) )
# #row[i] = newcol #str(col)
#else:
#newcol = str(col)
newcol = str(col).replace("\n", "")
newcol = newcol.replace(chr(10), "")
newcol = newcol.replace(chr(13), "")
#print(newcol)
newrow.append(newcol)
j += 1
#print(newrow)
csvwriter.writerow(newrow)
i += 1
csvfile.close()
def load_standard_datadict_headings(self):
self.meta_columns = []
self.meta_columns = ['column_name', 'start_pos', 'width', 'unique_id_yn', 'matchfield_yn', 'bigmatch_type', 'data_format', 'comments']
return self.meta_columns
def display_openfile_dialogs(self, container, default_filepath=''):
file_types = [('All files', '*'), ('Text files', '*.csv;*.txt')]
kw_fpath = {"bgcolor":self.bgcolor, "frame_width":"", "frame_height":"", "file_category":"datadict"}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(self.parent_window, self, self.controller, "Data dictionary file to load:", open_or_save_as, "DataDictToLoad", file_types, **kw_fpath)
self.filepathobj_load_from.display_view(container) #Display the dialog for user to select a data dict file
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(self.parent_window, self, self.controller, "Save data dictionary to file:", open_or_save_as, "DataDictToSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(container) #Display the dialog for user to Ave As... a new data dict file
def instantiate_view_object(self, container):
self.view_object = DataDict_View(container, self)
return self.view_object
def display_view(self, container=None):
#Most often, we will display the Data Dictionary form in the main class's BigCanvas.BigFrame "container".
print("In DataDict.Display_View, Type of Container is '%s'" % (str(type(container))) )
kw_dict = {"width":self.frame_width, "background":self.bgcolor, "borderwidth":2, "padx":3, "pady":3}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
#self.controller.clear_main_canvas()
if self.datadict_file_to_load_from:
#Function "read_datadict_file_into_arrays" extracts the contents of the Data Dictionary file and stores it in arrays. The arrays are them passed to TkEntry to be displayed.
print("\n" + "In DataDict.display_view(), about to run read_datadict_file_into_arrays() with '" + str(self.datadict_file_to_load_from) + "'" )
self.read_datadict_file_into_arrays(self.datadict_file_to_load_from)
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(container) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container, self.datadict_file_to_load_from)
self.display_user_buttons(container)
print("\n In DataDict.display_view(), about to call view_object.initUI().")
#Display the VIEW for this data dictionary
self.view_object.initUI(**kw_dict) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
self.view_object.display_datadict_in_grid(**kw_dict) #When first initiated, the View object will call this during its InitUI() method. But to refresh it afterwards, call display_datadict_in_grid() explicitly.
self.controller.refresh_main_canvas()
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(".tk") == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(self.button_frame, "DataDict_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(background=self.bgcolor)
#Note: the specified CSV data dictionary file is loaded automatically when the user selects it (because update_filepath() is called )
#self.btnLoadDictFile = Button(self.button_frame, text="Load Data Dictionary from CSV File", width=30, command=self.display_view)
#self.btnLoadDictFile.grid(row=0, column=0, sticky=W)
#self.btnLoadDictFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to load
self.btnSaveToDictFile = Button(self.button_frame, text="Save Data Dictionary to CSV File", width=30, command=self.write_datadict_from_entrygrid)
self.btnSaveToDictFile.grid(row=0, column=1, sticky=W)
self.btnSaveToDictFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to save as
def update_filepath(self, file_name_with_path='', callback_string='', alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
self.logobject.logit("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string), True, True )
if callback_string.lower().strip()[:4] == "load" or callback_string.lower().strip()[:4] == "open":
self.datadict_file_to_load_from = file_name_with_path
#if file_name_with_path: #Commented this out because btnLoadDictFile seems to be unnecessary - we automatically load the Data Dict
# self.btnLoadDictFile.config(state=NORMAL)
#Refresh the Data Dictionary view when the user selects a new DataDict file.
if self.datadict_file_to_load_from:
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid() #Remove all existing values from the grid
elif callback_string.lower().strip()[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.datadict_file_to_save_to = file_name_with_path
if self.datadict_file_to_save_to:
self.btnSaveToDictFile.config(state=NORMAL)
else:
self.btnSaveToDictFile.config(state=DISABLED)
self.update_master_paths(file_name_with_path)
def update_master_paths(self, file_name_with_path):
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
if self.mem_or_rec == "mem":
self.controller.mem_datadict_last_opened = file_name_with_path #The controller tracks last file opened of this type, so that when the user is again prompted to open the same type of file, we can default this value in.
if self.mem_or_rec == "rec":
self.controller.rec_datadict_last_opened = file_name_with_path #The controller tracks last file opened of this type, so that when the user is again prompted to open the same type of file, we can default this value in.
print("\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s" % (self.controller.dir_last_opened, self.controller.datadict_dir_last_opened, self.controller.rec_datadict_last_opened, self.controller.mem_datadict_last_opened) )
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
class Error_UI_Model():
debug = False
error_message = None #Plain text error message
error_list = [] #List of errors that have been logged by the controller (LIST OF DICTS - EACH DICT CONTAINS AN ERROR MESSAGE AND EXCEPTION OBJECTS)
error_dict = None #The ERROR DICT currently being processed
parent_window = None #Parent_wiondow is the TKinter object itself (often known as "root"
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
title = None
bgcolor = None
bgcolors = []
frame_width = None
frame_height = None
dedupe_single_file = False #Is this a dedupe of a single file, or a match of two different files?
#parmf_file = None
def __init__(self, parent_window, controller, error_list=["Unspecified error (ui model parm)"], title="Error", bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height):
self.parent_window = parent_window #Parent_wiondow is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
self.error_list = error_list #Error_list is a list of DICTS - each dict contains a plain text error message and exception objects)
now = datetime.datetime.now()
self.init_time = str(now.year) + str(now.month).rjust(2,'0') + str(now.day).rjust(2,'0') + ' ' + str(now.hour).rjust(2,'0') + ':' + str(now.minute).rjust(2,'0')
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\n\n____________________________________________________________________", True, True )
self.logobject.logit("%s In Error_UI_Model._init_: title=%s" % (self.init_time, title), True, True )
if title is not None:
self.title = title
else:
self.title = "Nothing"
if frame_width is not None:
self.frame_width = frame_width
if frame_height is not None:
self.frame_height = frame_height
#if show_view is not None:
# self.show_view = show_view
if bgcolor is not None:
self.bgcolor = bgcolor
else:
self.bgcolor = gl_frame_color
self.bgcolors = ["#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0", "#FDFFDF", "#DFFFF0"]
if self.error_message is not None:
self.logobject.logit("\nCalling handle_error with message '%s'" % (self.error_message) )
return
#*****************************************************************************************************************************************
#ERROR VIEW OBJECT(S)
#NOTE: VIEWS are typically invoked by calling this MODEL object. The following methods are handlers for instantiating VIEWS.
def instantiate_view_object(self, container, index):
#Instantiate VIEWS here.
error_view = Error_UI_View(container, self, index)
return error_view
def display_view(self, container, index, **kw):
error_view = self.instantiate_view_object(container, index)
print("Kwargs for error_view.display_view(): ")
self.logobject.logit(str(kw), True, True)
for key, value in kw.items():
self.logobject.logit("%s = %s" % (key, value), True, True )
self.logobject.logit("\n In error_ui_model.display_view(), calling error_view.initUI().", True, True)
error_view.initUI(**kw) #DISPLAY THE FRAME OBJECT ON SCREEN
return error_view
def display_views(self, container=None, howmany_views=None):
#Instantiate VIEWS here.
print("\n Top of error_view.display_views()")
if container is None:
container = self.controller.bigcanvas.bigframe
if not self.error_list:
self.error_list = ["Unspecified error (ui model display views)"]
if howmany_views is None:
howmany_views = len(self.error_list)
self.logobject.logit("\n Top of error_view.display_views()", True, True)
if True: #to do: validate the existence of error message before launching UI
for i in range(0, howmany_views):
bgcolor = self.bgcolors[i] #bgcolor = "#FDFFDF"
print("\n In Error_UI_Model.display_views(), calling display_view() for iteration #" + str(i) +". BgColor=" + bgcolor)
#DISPLAY THE ERROR DISPLAY SCREEN HERE:
self.display_view(container, i, width=self.frame_width, background=bgcolor, borderwidth=2, padx=3, pady=3)
else:
pass #
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(".tk") == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(self.button_frame, "error_view.display_user_buttons()")
else:
stackslot = 0
#Testing: show a button that will dump to the command window the current contents of all the screen CONTROLS.
self.button_frame.grid(row=stackslot, column=0, sticky=EW)
self.button_frame.config(background=self.bgcolor)
def update_message_region(self, text='', clear_after_ms=4000, **kw):
#if not text:
# text = "Uh-oh"
self.message_region.configure(text=text)
self.message_region.after(clear_after_ms, self.clear_message_region)
def clear_message_region(self):
self.message_region.configure(text="")
class Datafile_to_RDBMS_UI_Model():
debug = False
error_message = None
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
dxr = None #Instance of the Datafile_to_RDBMS() class
source_format = None #Flat text file by default
output_format = None #Sqlite3 by default
source_file = None #Name and path of the source file to be converted
output_file = None #Sqlite file (if sqlite is the RDBMS)
datadict_file = None
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
filepathobj_datadict = None #FilePath object to allow user to select a data dictionary (for parsing text into distinct columns)
converter = None #Object that handles the conversion
#conversion_func_name = None
csv_column_headers = [] #List of column headers in the data dictionary CSV
title = '' #Title to be displayed in the Frame object
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
btnLoadFile = None
btnSaveToFile = None
def __init__(
self,
parent_window,
controller,
source_format=None,
output_format=None,
source_file=None,
output_file=None,
show_view=None,
title='File converson',
**kw
): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
self.parent_window = parent_window #parent_window is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
now = datetime.datetime.now()
self.init_time = str(now.year) + str(now.month).rjust(
2, '0') + str(now.day).rjust(2, '0') + ' ' + str(now.hour).rjust(
2, '0') + ':' + str(now.minute).rjust(2, '0')
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit(
"\n\n____________________________________________________________________",
True, True)
self.logobject.logit(
"\nIn Datafile_to_RDBMS_UI_Model._init_: source_file='%s' -- and type(controller)=%s"
% (source_file, type(controller)), True, True)
self.source_format = source_format
self.output_format = output_format
self.source_file = source_file
self.output_file = output_file
self.title = title
self.show_view = show_view
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor = gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width = gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height = gl_frame_height
def set_source_format(self, source_format):
self.source_format = source_format
print("\nSetting the source_format to: %s" % (source_format))
def set_output_format(self, output_format):
self.output_format = output_format
print("\nSetting the output_format to: %s" % (output_format))
def convert_file(self, source_format=None, output_format=None):
'''Called after the user has selected a source file and an output file '''
if not source_format:
source_format = self.source_format
if not output_format:
output_format = self.output_format
source_format = source_format.lower().strip()
output_format = output_format.lower().strip()
#Based on source and output formats, call the appropriate function to handle the conversion.
if source_format == "txt" and output_format[:6] == "sqlite":
self.convert_txt_to_sqlite(self.source_file, self.output_file)
def convert_txt_to_sqlite(self, source_file=None, output_file=None):
'''DOCSTRING '''
if not source_file:
source_file = self.source_file
if not output_file:
output_file = self.output_file
#Instantiate the Datafile_to_RDBMS class
self.converter = Datafile_to_RDBMS()
self.logobject.logit(
"\nAbout to convert file '%s' to table in RDBMS '%s'" %
(source_file, output_file), True, True)
self.converter.datafile = source_file
head, tail = os.path.split(source_file)
db_tablename = tail.lower().strip()
dotpos = db_tablename.find(".")
if (dotpos != -1):
db_tablename = db_tablename[:dotpos].replace(".", "")
self.converter.db_tablename = db_tablename
self.converter.db_catalog = output_file
self.converter.db_platform = "sqlite3"
self.logobject.logit(
"\n Type of self.converter: %s" % (str(type(self.converter))),
True, True)
#if(str(type(self.converter)).lower().find("datafile_to_rdbms") == -1): #When the class is first instantiated, load the data dictionary
if len(self.converter.datadict) == 0:
self.copy_datadict_to_class_properties()
self.converter.display_datadict()
#*******************************************
self.converter.import_data()
#*******************************************
return self.converter.error_message
def instantiate_view_object(self, container):
self.view_object = Datafile_to_RDBMS_UI_View(container, self)
return self.view_object
def copy_datadict_to_class_properties(self):
datadict = Datadict_Common() #Standard DataDict class
hdr_list = datadict.load_standard_datadict_headings(
["bigmatch"]
) #Make sure we are using the standard, updated list of column headings for the Data Dictionary
datadict = None #Erase the class instantiation when done to release memory
#Check our assumptions about which Data Dictionary column headings are in the standard:
if not "column_name" in hdr_list:
self.error_message = "Expected item 'column_name' to be in Data Dictionary header row"
if not "start_pos" in hdr_list:
self.error_message = "Expected item 'start_pos' to be in Data Dictionary header row"
if not "width" in hdr_list:
self.error_message = "Expected item 'width' to be in Data Dictionary header row"
if self.error_message:
print("\n \n ERROR: " + self.error_message)
try:
#Read the Data Dictionary and store information for each COLUMN as properties of the
self.logobject.logit("\nAdding columns to the dxr.DataDictionary",
True, True)
with open(self.datadict_file, 'r') as csvfile:
csvreader = csv.reader(csvfile,
delimiter=',') #, quotechar='')
count = 0
for row in csvreader:
if count == 0: #Top row of CSV
self.build_column_header_list(row)
break
#Ascertain the correct position within the Data Dictionary where column name, width and startpos are stored
pos_colname = int(
self.get_position_of_datadict_column("column_name"))
pos_width = int(self.get_position_of_datadict_column("width"))
pos_startpos = int(
self.get_position_of_datadict_column("start_pos"))
if self.debug:
print("\npos_colname=%s, pos_width=%s, pos_startpos=%s" %
(pos_colname, pos_width, pos_startpos))
count = 0
for row in csvreader:
#NOTE: CSV reader is a forward-only reader -- and it has already read row 0, the top row. So now the next row WILL BE ROW 1!
#Add this column from the data dictionary to the list of columns that will be created in the RDBMS
#Determine whether this is a blank row in the file
if self.row_is_empty(row):
continue
if self.debug:
self.logobject.logit(
"Dict row " + str(count) + ": " + row[0] + ", " +
row[1] + ", " + row[2], True, True, True)
self.converter.add_column(row[pos_colname], "char",
row[pos_width],
row[pos_startpos], "file")
count += 1
csvfile.close()
except:
self.controller.common.handle_error(self.error_message, False,
False, "datafile_rdbms_ui")
if self.debug: raise
def get_position_of_datadict_column(self, which_column):
which_column = which_column.lower().strip()
returnval = ""
for hdr in self.csv_column_headers:
self.logobject.logit(
"Seeking datadict column %s... Found: %s" %
(which_column, hdr["col_hdr"].lower().strip()), True, True)
if hdr["col_hdr"].lower().strip() == which_column:
returnval = hdr["col_index"]
break
self.logobject.logit(
"Seeking datadict column '%s'... Found position: %s" %
(which_column, returnval), True, True)
return returnval
def build_column_header_list(self, hdr_row):
col_index = 0
for col_hdr in hdr_row:
col_hdr = col_hdr.strip()
self.logobject.logit("(%s) col_hdr: %s" % (col_index, col_hdr),
True, True)
temp = {
"col_index": col_index,
"col_hdr": col_hdr,
"max_width": 0,
"start_pos": 0,
"data_type": "",
"selected": ""
}
self.csv_column_headers.append(temp)
col_index += 1
def display_view(self, container=None):
#Most often, we will display the form in the main class's BigCanvas.BigFrame "container".
self.logobject.logit(
"In DataFile_RDBMS_UI.Display_View, Type of Container is '%s'" %
(str(type(container))), True, True)
kw_dict = {
"width": self.frame_width,
"background": self.bgcolor,
"borderwidth": 2,
"padx": 3,
"pady": 3
}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(
container
) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container, self.source_file)
self.display_user_buttons(container)
#print("\n In Datafile_to_RDBMS_UI_Model.display_view(), about to call view_object.initUI().")
#Display the VIEW for this data dictionary
#self.view_object.initUI(**kw_dict) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
#self.view_object.display_datadict_in_grid(**kw_dict) #When first initiated, the View object will call this during its InitUI() method. But to refresh it afterwards, call display_datadict_in_grid() explicitly.
self.controller.refresh_main_canvas()
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(
".tk"
) == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(
self.button_frame,
"Datfile_to_RDBMS_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(background=self.bgcolor)
#Button to launch the conversion process:
self.btnSaveToFile = Button(
self.button_frame,
text="Convert File",
width=30,
command=self.convert_file) #convert_txt_to_sqlite
self.btnSaveToFile.grid(row=0, column=1, sticky=W)
self.btnSaveToFile.config(
state=DISABLED
) #Do not enable this button unless the user has selected a Data Dictionary file to save as
def display_openfile_dialogs(self, container, default_filepath=''):
file_types = [('All files', '*'), ('SAS files', '*.sas7bdat; *.txt')]
kw_fpath = {
"bgcolor": self.bgcolor,
"frame_width": "",
"frame_height": "",
"file_category": "datadict"
}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(
self.parent_window, self, self.controller, "Source file to load:",
open_or_save_as, "ConvertSourceToLoad", file_types, **kw_fpath)
self.filepathobj_load_from.display_view(
container) #Display the dialog for user to select a data dict file
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(
self.parent_window, self, self.controller, "Sqlite .DB file:",
open_or_save_as, "ConvertOutputToSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(
container
) #Display the dialog for user to Save As... a new data dict file
open_or_save_as = "open"
self.filepathobj_datadict = FilePath_Model(
self.parent_window, self, self.controller, "Data dictionary:",
open_or_save_as, "DatadictToLoad", file_types, **kw_fpath)
self.filepathobj_datadict.display_view(
container) #Display the Open File dialog
def update_filepath(self,
file_name_with_path='',
callback_string='',
alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
#self.logobject.logit("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string), True, True )
print(
"Master Datafile_to_RDBMS_UI_Model object has gotten the alert: filename is %s and callback_string is '%s'"
% (file_name_with_path, callback_string))
if callback_string.lower().strip(
)[:4] == "load" or callback_string.lower().strip()[:4] == "open":
if str(callback_string).lower()[4:].find(
"datadict") != -1: #User selected a DataDictionary file
self.logobject.logit(
"datadict_file is being set to %s" % (file_name_with_path),
True, True)
self.datadict_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.source_file because of the "callback string" returned by the FilePath object.
if self.datadict_file: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid(
) #Remove all existing values from the grid
else: #User selected the Source File
self.source_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.source_file because of the "callback string" returned by the FilePath object.
if self.source_file: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid(
) #Remove all existing values from the grid
elif callback_string.lower().strip(
)[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.output_file = file_name_with_path
if self.output_file:
self.btnSaveToFile.config(state=NORMAL)
else:
self.btnSaveToFile.config(state=DISABLED)
self.update_master_paths(file_name_with_path)
self.update_initial_dir_for_file_open_dialogs()
def update_master_paths(self, file_name_with_path):
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
print(
"\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s"
% (self.controller.dir_last_opened,
self.controller.datadict_dir_last_opened,
self.controller.rec_datadict_last_opened,
self.controller.mem_datadict_last_opened))
def update_initial_dir_for_file_open_dialogs(self):
'''In addition to tracking "last file opened" at the main controller level, we also want to notify every FilePath object when the user has opened a new file, so that they can adjust thir Initial DIr properties to the location just opened.'''
self.filepathobj_load_from.calc_initial_dir_for_file_open(
self.source_file, "datadict", "")
self.filepathobj_save_to.calc_initial_dir_for_file_open(
self.output_file, "datadict", "")
self.filepathobj_datadict.calc_initial_dir_for_file_open(
self.datadict_file, "datadict", "")
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found
def row_is_empty(self, row):
isempty = False
rowcheck = ','.join(row)
rowcheck = rowcheck.replace(",", "").replace("'", "").replace(
"\n", "").replace("?", "").strip()
#print("Rowcheck: %s" % (rowcheck))
if rowcheck == "":
#print("ROW IS EMPTY. SKIP IT.")
isempty = True
return isempty
class ConvertFile_Model():
debug = False
error_message = None
controller = None #Controller is the BigMatchController class in main.py
logobject = None #Instantiation of CHLog class
source_format = None
output_format = None
source_file = None #Name and path of the source file to be converted
output_file = None #Name and path of the file to be created (output)
datadict_file = None
filepathobj_load_from = None #FilePath object to allow user to select a file
filepathobj_save_to = None #FilePath object to allow user to select a file
filepathobj_datadict = None #FilePath object to allow user to select a data dictionary (for parsing text into distinct columns)
converter = None
#conversion_func_name = None
title = '' #Title to be displayed in the Frame object
bgcolor = 'ivory' #Background color of the Frame widget that displays the Data Dictionary contents
frame_width = None
frame_height = None
view_object = None
show_view = None
btnLoadFile = None
btnSaveToFile = None
def __init__(self, parent_window, controller, source_format=None, output_format=None, source_file=None, output_file=None, show_view=None, title='File converson', **kw): #, bgcolor=gl_frame_color, frame_width=gl_frame_width, frame_height=gl_frame_height
self.parent_window = parent_window #parent_window is the TKinter object itself (often known as "root"
self.controller = controller #Controller is the BigMatchController class in main.py
print("\nIn ConvertFile._init_, source_file = '" + str(self.source_file) + "' -- and show_view=" + str(self.show_view))
self.logobject = CHLog(self.controller.enable_logging)
self.logobject.logit("\nIn ConvertFile._init_: source_file = '" + str(self.source_file) + "' -- and show_view=" + str(self.show_view), True, True )
self.source_format = source_format
self.output_format = output_format
self.source_file = source_file
self.output_file = output_file
self.title = title
self.show_view = show_view
if self.check_key_exists("bgcolor", **kw):
self.bgcolor = kw["bgcolor"]
if not self.bgcolor:
self.bgcolor=gl_frame_color
if self.check_key_exists("frame_width", **kw):
self.frame_width = kw["frame_width"]
if not self.frame_width:
self.frame_width=gl_frame_width
if self.check_key_exists("frame_height", **kw):
self.frame_height = kw["frame_height"]
if not self.frame_height:
frame_height=gl_frame_height
def set_source_format(self, source_format):
self.source_format = source_format
print("\nSetting the source_format to: %s" % (source_format) )
def set_output_format(self, output_format):
self.output_format = output_format
print("\nSetting the output_format to: %s" % (output_format) )
def convert_file(self, source_format=None, output_format=None):
'''Call either convert_sas_to_text() or convert_csv_to_text(). This function is called AFTER the user has selected a source file and an output file. '''
if not source_format:
source_format = self.source_format
if not output_format:
output_format = self.output_format
source_format = source_format.lower().strip()
output_format = output_format.lower().strip()
#Based on source and output formats, call the appropriate function to handle the conversion.
if source_format == "sas" and output_format == "text":
self.convert_sas_to_text(self.source_file, self.output_file)
elif source_format == "csv" and (output_format == "text" or output_format == "txt" or output_format == "flat"):
self.convert_csv_to_text(self.source_file, self.output_file, self.datadict_file)
def convert_sas_to_text(self, source_file=None, output_file=None):
'''DOCSTRING '''
if not source_file:
source_file = self.source_file
if not output_file:
output_file = self.output_file
self.converter = SAS7BDAT(source_file)
print("\nAbout to convert SAS file '%s' to text file '%s'" % (source_file, output_file) )
self.converter.convertFile(output_file, '\t')
def convert_csv_to_text(self, source_file=None, output_file=None, datadict_file=None):
'''DOCSTRING '''
if not source_file:
source_file = self.source_file
if not output_file:
output_file = self.output_file
if not datadict_file:
datadict_file = self.datadict_file
top_row_is_header = False
if datadict_file:
print("\nIn ConvertFile, datadict_file is: %s" % (datadict_file) )
#top_row_is_header = True #TO DO: allow user to check a checkbox indicating that the top CSV row contains column headers. FOR NOW--If we have a Data Dictionary, then the assumption is that the columns have names. But this is not always the case.
if python_common_found:
self.converter = TextFile(self.parent_window, self.controller)
print("\nAbout to convert CSV file '%s' to text file '%s'" % (source_file, output_file) )
self.converter.convert_csv_to_flat_text(source_file, output_file, datadict_file, top_row_is_header)
def instantiate_view_object(self, container):
self.view_object = ConvertFile_View(container, self)
return self.view_object
def display_view(self, container=None):
#Most often, we will display the form in the main class's BigCanvas.BigFrame "container".
print("In ConvertFile.Display_View, Type of Container is '%s'" % (str(type(container))) )
kw_dict = {"width":self.frame_width, "background":self.bgcolor, "borderwidth":2, "padx":3, "pady":3}
if container == None:
container = self.controller.bigcanvas.bigframe #This is the default canvas/frame object on which all other widgets are displayed.
if self.view_object is None: #We have not yet instantiated the View. Make sure to display any Open File dialogs at top before rendering other views.
self.instantiate_view_object(container) #Just in case we need to run this module outside the parent frame during testing.
#Display a file open dialog so the user can point us to the Data Dictionary they'd like to open:
self.display_openfile_dialogs(container, self.source_file)
self.display_user_buttons(container)
#print("\n In ConvertFile.display_view(), about to call view_object.initUI().")
#Display the VIEW for this data dictionary
#self.view_object.initUI(**kw_dict) #width=self.frame_width, background=self.bgcolor, borderwidth=2, padx=3, pady=3)
else:
#self.view_object.display_datadict_in_grid(**kw_dict) #When first initiated, the View object will call this during its InitUI() method. But to refresh it afterwards, call display_datadict_in_grid() explicitly.
self.controller.refresh_main_canvas()
def display_user_buttons(self, container):
'''Function display_user_buttons shows one or more buttons near top of page for common user functions, so the user doesn't need to constantly hit the system menus. '''
self.button_frame = Frame(container)
if str(type(container)).lower().find(".tk") == -1: #For testing, we might display this object directly in the Tkinter main window. If this is the case, then don't call get_widget_position().
stackslot = container.get_widget_position(self.button_frame, "ConvertFile_Model.display_user_buttons()")
else:
stackslot = 0
self.button_frame.grid(row=stackslot, column=0, sticky=W)
self.button_frame.config(background=self.bgcolor)
#Button to launch the conversion process:
self.btnSaveToFile = Button(self.button_frame, text="Convert File", width=30, command=self.convert_file) #convert_sas_to_text
self.btnSaveToFile.grid(row=0, column=1, sticky=W)
self.btnSaveToFile.config(state=DISABLED) #Do not enable this button unless the user has selected a Data Dictionary file to save as
def display_openfile_dialogs(self, container, default_filepath=''):
file_types = [('All files', '*'), ('SAS files', '*.sas7bdat; *.txt')]
kw_fpath = {"bgcolor":self.bgcolor, "frame_width":"", "frame_height":"", "file_category":"datadict"}
open_or_save_as = "open"
self.filepathobj_load_from = FilePath_Model(self.parent_window, self, self.controller, "Source file to load:", open_or_save_as, "ConvertSourceToLoad", file_types, **kw_fpath)
self.filepathobj_load_from.display_view(container) #Display the dialog for user to select a data dict file
open_or_save_as = "save_as"
self.filepathobj_save_to = FilePath_Model(self.parent_window, self, self.controller, "Save file to:", open_or_save_as, "ConvertOutputToSaveAs", file_types, **kw_fpath)
self.filepathobj_save_to.display_view(container) #Display the dialog for user to Ave As... a new data dict file
open_or_save_as = "open"
self.filepathobj_datadict = FilePath_Model(self.parent_window, self, self.controller, "Data dictionary (optional):", open_or_save_as, "DatadictToLoad", file_types, **kw_fpath)
self.filepathobj_datadict.display_view(container) #Display the Open File dialog
def update_filepath(self, file_name_with_path='', callback_string='', alias=''):
'''IMPORTANT: ALL FilePath objects created by this class will expect Function "update_file_path" to exist! FilePath objects alert their masters when a filepath is selected in an open-file dialog.'''
#self.logobject.logit("Master DataDict_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string), True, True )
print("Master Datafile_to_RDBMS_UI_Model object has gotten the alert: filename is %s and callback_string is '%s'" % (file_name_with_path, callback_string))
if callback_string.lower().strip()[:4] == "load" or callback_string.lower().strip()[:4] == "open":
if str(callback_string).lower()[4:].find("datadict") != -1: #User selected a DataDictionary file
self.logobject.logit("datadict_file is being set to %s" % (file_name_with_path), True, True )
self.datadict_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.source_file because of the "callback string" returned by the FilePath object.
if self.datadict_file: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid() #Remove all existing values from the grid
else: #User selected the Source File
self.source_file = file_name_with_path #file_name_with_path is the name/path of the file selected by the user. We know to store this to self.source_file because of the "callback string" returned by the FilePath object.
if self.source_file: #Refresh the view when the user selects a new file.
self.display_view()
else: #No file was specified (user might have cleared out a previously selected file name)
self.view_object.clear_datadict_grid() #Remove all existing values from the grid
elif callback_string.lower().strip()[:4] == "save": #This is a file SAVE AS, not a FILE OPEN
self.output_file = file_name_with_path
if self.output_file:
self.btnSaveToFile.config(state=NORMAL)
else:
self.btnSaveToFile.config(state=DISABLED)
self.update_master_paths(file_name_with_path)
self.update_initial_dir_for_file_open_dialogs()
def update_master_paths(self, file_name_with_path):
if file_name_with_path:
head, tail = os.path.split(file_name_with_path)
self.controller.dir_last_opened = head
self.controller.datadict_dir_last_opened = head #The controller tracks last folders opened for this type, so that when the user is again prompted to open the same type of file, we can set this as the initial dir.
print("\n Controller-saved paths-- LastDir: %s, LastDataDictDir: %s, LastRecDatadict: %s, LastMemDatadict: %s" % (self.controller.dir_last_opened, self.controller.datadict_dir_last_opened, self.controller.rec_datadict_last_opened, self.controller.mem_datadict_last_opened) )
def update_initial_dir_for_file_open_dialogs(self):
'''In addition to tracking "last file opened" at the main controller level, we also want to notify every FilePath object when the user has opened a new file, so that they can adjust thir Initial DIr properties to the location just opened.'''
self.filepathobj_load_from.calc_initial_dir_for_file_open(self.source_file, "datadict", "mem")
self.filepathobj_save_to.calc_initial_dir_for_file_open(self.output_file, "datadict", "rec")
self.filepathobj_datadict.calc_initial_dir_for_file_open(self.datadict_file, "datadict", "")
def check_key_exists(self, keyvalue, **kw):
found = False
#print("Checking for key '%s' in **Kwargs" % (keyvalue) )
for key, value in kw.items():
if str(key).lower() == str(keyvalue).lower():
found = True
break
#print("Checking for key '%s' in **Kwargs -- Found? %s" % (str(keyvalue), str(found) ) )
return found