def validate_data(csv_file, schema_descr): """Tries to validate data in csv file using schema description. Any errors that are found, are saved in file consts['validator_errors_name']. Returns True if no errors were found, otherwise False. Arguments: csv_file -- CsvFile object representing data schema_descr -- schema describing fields in collection """ csv_file.reset() csv_data = CsvData(csv_file) validator = DataValidator(csv_data, schema_descr) validator.check_all() correct = validator.is_all_correct() if correct: print 'No errors have been found during validation.' else: print 'Errors have been found during validation.' errors_file_name = consts['validator_errors_name'] print 'Saving errors to file %s' % errors_file_name try: errors_file = open(errors_file_name, 'wb') except IOError: print 'Can not open file %s, no info will be saved.' % errors_file_name else: errors_file.write(validator.get_errors_log().encode('utf-8')) return correct
class TestDataValidator(unittest.TestCase): def setUp(self): self.vld = DataValidator() def test_invalid_empid(self): data = ['A0011', 'F', '21', '001', 'Normal', '12', '1-1-1996'] result = [None, 'F', '21', '001', 'Normal', '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_invalid_gender(self): data = ['A001', '222', '21', '001', 'Normal', '12', '1-1-1996'] result = ['A001', None, '21', '001', 'Normal', '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_valid_gender_01(self): data = ['A001', 'girl', '21', '001', 'Normal', '12', '1-1-1996'] result = ['A001', 'F', '21', '001', 'Normal', '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_invalid_age(self): data = ['A001', 'M', '101', '001', 'Normal', '12', '1-1-1996'] result = ['A001', 'M', None, '001', 'Normal', '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_invalid_sales(self): data = ['A001', 'M', '21', 'abc', 'Normal', '12', '1-1-1996'] result = ['A001', 'M', '21', None, 'Normal', '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_invalid_bmi(self): data = ['A001', 'M', '21', '001', 'Thin', '12', '1-1-1996'] result = ['A001', 'M', '21', '001', None, '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_invalid_salary(self): data = ['A001', 'M', '21', '001', 'Normal', '1', '1-1-1996'] result = ['A001', 'M', '21', '001', 'Normal', None, '01-01-1996'] self.assertEqual(self.vld.check_all(data), result) def test_invalid_birthday_01(self): data = ['A001', 'M', '21', '001', 'Normal', '12', '31-2-1996'] self.assertRaises(ValueError, self.vld.check_all, data) def test_invalid_birthday_02(self): data = ['A001', 'M', '21', '001', 'Normal', '12', '15-May-1996'] result = ['A001', 'M', '21', '001', 'Normal', '012', None] self.assertEqual(self.vld.check_all(data), result) def test_invalid_values(self): data = ['A001', 'M', '21', '001', 'Normal', '12'] self.assertEqual(self.vld.check_all(data), []) def test_valid_values(self): data = ['A001', 'M', '21', '001', 'Normal', '12', '1-1-1996'] result = ['A001', 'M', '21', '001', 'Normal', '012', '01-01-1996'] self.assertEqual(self.vld.check_all(data), result)
def get_bmi(): """ >>> get_bmi() {'Obesity': 1, 'Normal': 2} """ sd = EmployeeData() v = DataValidator() sd.add_data( v.check_all(["T133", "M", "28", "100", "Obesity", "350", "11-09-1990"])) sd.add_data( v.check_all(["T134", "M", "29", "150", "Normal", "300", "12-08-1989"])) sd.add_data( v.check_all(["T135", "F", "30", "100", "Normal", "300", "13-07-1988"])) return sd.get_bmi()
def get_gender(): """ >>> get_gender() {'Male': 2, 'Female': 1} """ sd = StaffData() v = DataValidator() sd.add_data( v.check_all(["T133", "M", "28", "100", "Obesity", "350", "11-09-1990"])) sd.add_data( v.check_all(["T134", "M", "29", "150", "Normal", "300", "12-08-1989"])) sd.add_data( v.check_all(["T135", "F", "30", "100", "Normal", "300", "13-07-1988"])) return sd.get_gender()
def validate_all(raw_data): """ >>> validate_all(["T123", "M", "28", "100", "heisobesity", "$350", "11-09-1990"]) ['T123', 'M', '28', 100, 'Obesity', 350, '11-09-1990'] """ v = DataValidator() return v.check_all(raw_data)
class Controller(Cmd): def __init__(self): Cmd.__init__(self) # The command line indicator self.prompt = ">>> " # Welcome information self.intro = "Welcome to Staff Information System\nEnter a keyword to start. For help, enter \"help\"." # Object of DataValidator, for validating data self._vld = DataValidator() # Instance of StaffData self._std = StaffData() def do_select(self, line): """ Select a data source :param line: <String> Source name :return: None :Author: Zhiming Liu """ # Available data sources options = "-csv", "-db" args = list(arg.lower() for arg in str(line).split()) try: # Check if the input data source is available in this program or not if args[0] not in options: raise ValueError("The data resource is not available.") else: # Code for initialise CSV data source if args[0] == "-csv": try: if len(args) == 1: self._std.select_source(args[0][1:], "staffinfo.csv") View.warning( "No CSV file path specified. A default file \"staffinfo.csv\" will be used." ) elif len(args) == 2: self._std.select_source(args[0][1:], args[1]) elif len(args) == 3: if args[1] == "-a": self._std.select_source( args[0][1:], args[2], True) except (CSVError, OSError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data source CSV is selected.") # Code for initialise database source elif args[0] == "-db": try: self._std.select_source(args[0][1:]) except (ConnectionError, TypeError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data source Database is selected.") # Code for initialise XXXX data source else: pass # Catch and display error message except ValueError as e: View.error(str(e) + "\n") View.help_select() except Exception as e: View.error(e) def do_add(self, line): """ Add a new entry of data :param line: <EMPID> <Age> <Gender> <Sales> <BMI> <Salary> <Birthday> :return: None """ # Split the input argument to obtain the data raw_data = list(arg.lower() for arg in str(line).split()) try: # Check if input data has 7 data fields if not len(raw_data) == len(Data): raise AttributeError("Please input correct data.") else: # Check and wash data by check_all() of DataValidator result = self._vld.check_all(raw_data) # Check if there is any None which stands for invalid input if None in result: key = 0 # build a list of name list items = list(map(lambda i: i.name, Data)) e_str = "" while key < len(result): if result[key] is None: # Left alignment e_str += "{:<10}".format(items[key]) key += 1 raise ValueError("The following field(s) is invalid:\n%s" % e_str) else: self._std.add_data(result) except (AttributeError, ValueError) as e: View.error(str(e) + "\n") View.help_add() except CSVError as e: View.error(e) except Exception as e: View.error(e) else: View.success("Add data") def do_save(self, arg): """ Save data to specified data source :param arg: arg :return: None """ # If no data source selected, prompt user to do so. try: self._std.save_data() except ValueError as e: View.info(e) except (OSError, AttributeError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data is saved") def do_show(self, line): # Get all instructions args = list(arg.lower() for arg in str(line).split()) # Those commands are required single arguments # single_commands = ["-a"] # Those commands are required two arguments plot_commands = ["-p", "-b", "-c"] # Show data table if args[0] == "-t": if len(self._std.data) == 0 and len(self._std.new_data) == 0: View.info("No data to display.") if not len(self._std.data) == 0: View.display("ORIGINAL DATA:") View.display_data(self._std.data, ind=True) if not len(self._std.new_data) == 0: View.display("\nNEW DATA:") View.display_data(self._std.new_data, ind=True) View.display("\n(Input command \"save\" to save the new data)") elif args[0] in plot_commands: try: if len(args) == 1: raise IndexError("Incomplete command line.") if args[0] == "-p": self.show_pie(args[1]) if args[0] == "-b": self.show_bar(args[1]) if args[0] == "-c": self.show_scatter(args[1]) except IndexError as e: View.error(str(e) + "\n") View.help_show() else: View.info("Invalid command line.\n") View.help_show() def show_pie(self, line): # Draw Pies try: if len(self._std.get_gender()) == 0 or len( self._std.get_bmi()) == 0: raise ValueError("No data to display.") # Draw gender if line.upper() == Data.GENDER.name: View.plot_pie(self._std.get_gender(), "Gender Distribution") # Draw BMI if line.upper() == Data.BMI.name: View.plot_pie(self._std.get_bmi(), "Body Mass Index (BMI)") except ValueError as e: View.info(e) except Exception as e: View.error(e) def show_bar(self, line): # Draw Bars try: if len(self._std.get_gender()) == 0 or len( self._std.get_gender()) == 0: raise ValueError("No data to display.") # Draw gender if line.upper() == Data.GENDER.name: View.plot_bar(self._std.get_gender(), "Gender Distribution") # Draw BMI if line.upper() == Data.BMI.name: View.plot_bar(self._std.get_bmi(), "Body Mass Index (BMI)") except ValueError as e: View.info(e) except Exception as e: View.error(e) # Author: Vaishali Patel def show_Scatter(self, line): # Draw Line Scatter try: if len(self._std.get_gender()) == 0 or len( self._std.get_gender()) == 0: raise ValueError("No data to display.") # Draw gender if line.upper() == Data.GENDER.name: View.plot_bar(self._std.get_gender(), "Gender Distribution") # Draw BMI if line.upper() == Data.SALARY.name: View.plot_bar(self._std.get_salary(), "Body Mass Index (SALARY)") except ValueError as e: View.info(e) except Exception as e: View.error(e) @staticmethod def help_show(): View.help_show() @staticmethod def help_add(): View.help_add() @staticmethod def help_save(): View.help_save() @staticmethod def help_select(): View.help_select() @staticmethod def help_quit(): View.help_quit() def do_quit(self, line): arg = str(line).lower() if not arg == "-f" and not len(self._std.new_data) == 0: View.warning( "The new data hasn't been saved. Enter \"quit -f\" to quit without saving." ) else: View.display("Thanks for using. Bye!") return True
class InterpreterController(Cmd): _input_file = '' _output_file = '' _data_list = [] # _graph = None # This class is a controller class which enable interpreter (model) and # View (Console View) to interconnect with each other and # This class also defined command loop enables \ # the associate commands invoke # # Written By: Patel # def __init__(self): Cmd.__init__(self) # Initialize cmd interface here self.prompt = ">>> " # Initialize prompt self._vld = DataValidator( ) # Object of DataValidator, for validating data # self._graph = GraphDirector() # self._grp = Graph() self._graph = ShowGraph() self._shw = GetEmployee() # Instace of GetEmployeeData self.intro = "WELCOME TO THE EMPLOYEE DATABASE MANAGEMENT CONSOLE \n ENTER A KEYWORD TO START. FOT HELP, ENTER \"help\"." # Welcome information def do_select(self, line): """ Select a data source :param line: <String> Source name :return: None :Author: Vaishali Patel """ # Available data sources options = "-csv", "-db" args = list(arg.lower() for arg in str(line).split()) try: # Check if the input data source is available in this program or not if args[0] not in options: raise ValueError("The data resource is not available.") else: # Code for initialise CSV data source if args[0] == "-csv": try: if len(args) == 1: self._shw.select_source(args[0][1:], "employeeinfo.csv") View.warning( "No CSV file path specified. A default file \"employeeinfo.csv\" will be used." ) elif len(args) == 2: self._shw.select_source(args[0][1:], args[1]) elif len(args) == 3: if args[1] == "-a": self._shw.select_source( args[0][1:], args[2], True) except (CSVError, OSError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data source CSV is selected.") # Code for initialise database source elif args[0] == "-db": try: self._shw.select_source(args[0][1:]) except (ConnectionError, TypeError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data source Database is selected.") # Code for initialise XXXX data source else: pass # Catch and display error message except ValueError as e: View.error(str(e) + "\n") View.help_select() except Exception as e: View.error(e) def do_add(self, line): """ # This function enable to add employee data into system # Which basically give all the available option to the user # parse: splits the given sequence of characters or values (text) # into smaller parts based on some rules # # Written By: Patel :param line: <EMPID> <Age> <Gender> <Sales> <BMI> <Salary> <Birthday> :return: None """ # Split the input argument to obtain the data raw_data = list(arg.lower() for arg in str(line).split()) try: # Check if input data has 7 data fields if not len(raw_data) == len(Data): raise AttributeError("Please input correct data.") else: # Check and wash data by check_all() of DataValidator result = self._vld.check_all(raw_data) # Check if there is any None which stands for invalid input if None in result: key = 0 # build a list of name list items = list(map(lambda i: i.name, Data)) e_str = "" while key < len(result): if result[key] is None: # Left alignment e_str += "{:<10}".format(items[key]) key += 1 raise ValueError("The following field(s) is invalid:\n%s" % e_str) else: self._shw.add_data(result) except (AttributeError, ValueError) as e: View.error(str(e) + "\n") View.help_add() except CSVError as e: View.error(e) except Exception as e: View.error(e) else: View.success("Add data") def do_save(self, arg): """ Save data to specified data source# This function enable to access dict option for respective file # to be stored in the system # Raise Exception error if unable to do so # # Written By: Patel # :param arg: arg :return: None """ # If no data source selected, prompt user to do so. try: self._shw.save_data() except ValueError as e: View.info(e) except (OSError, AttributeError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data is saved") def do_show(self, line): # Get all instructions# This function enable to create dictionary option # to respective commands and the system find matches # to the respective option to show individual map / view # # Written By: Patel # args = list(arg.lower() for arg in str(line).split()) # Those commands are required single arguments # single_commands = ["-a"] # Those commands are required two arguments # Show data table if args[0] == "-t": if len(self._shw.data) == 0 and len(self._shw.new_data) == 0: View.info("No data to display.") if not len(self._shw.data) == 0: View.display("ORIGINAL DATA:") View.display_data(self._shw.data, ind=True) if not len(self._shw.new_data) == 0: View.display("\nNEW DATA:") View.display_data(self._shw.new_data, ind=True) View.display("\n(Input command \"save\" to save the new data)") else: View.info("Invalid command line.\n") View.help_show() def do_graph_show(self, line): _graph = None graph_director = GraphDirector(None) plot_commands = ["-p", "-b", "-c"] if args[0] in plot_commands: try: if len(args) == 1: raise IndexError("Incomplete command line.") if args[0] == "-p": graph_builder = self._graph.show_pie(self._data_list) graph_director.set_builder(graph_builder) if args[0] == "-b": graph_builder = self._graph.show_bar(self._data_list) graph_director.set_builder(graph_builder) if args[0] == "-c": graph_builder = self._graph.show_scatter(self._data_list) graph_director.set_builder(graph_builder) except IndexError as e: View.error(str(e) + "\n") View.help_show() else: View.info("Invalid command line.\n") View.help_show() graph_director.construct() _graph = graph_builder.get_graph() return _graph @staticmethod def help_show(): View.help_show() @staticmethod def help_add(): View.help_add() @staticmethod def help_save(): View.help_save() @staticmethod def help_select(): View.help_select() @staticmethod def help_quit(): View.help_quit() def do_quit(self, line): arg = str(line).lower() if not arg == "-f" and not len(self._shw.new_data) == 0: View.warning( "The new data hasn't been saved. Enter \"quit -f\" to quit without saving." ) else: View.display("Thanks for using. Bye!") return True def check_valid_file_name(self, file_name, input_output): """ Checks to make sure the file_name exists and if it has a valid file extension :param file_name: :param input_output: :return: """ if self.check_file_exists(file_name): if self.check_file_name_extensions(file_name, input_output): return True else: return False else: print("File does not exist") return False def check_file_name_extensions(self, file_name, input_output): """ Checks the file extension against the possible extensions set up in the class variable _extension_types :param file_name: :param input_output: :return: """ file_type = FileTypes() extension_types = file_type.get_extension_types() for extension in extension_types: if file_name.endswith(extension): if input_output == 'input': self._input_file = file_type.get_file_type(extension) else: self._output_file = file_type.get_file_type(extension) return True print("File name must end with:") for extension in extension_types: print(extension) return False
class Controller(Cmd): def __init__(self): Cmd.__init__(self) # The command line indicator self.prompt = ">>> " # Welcome information self.intro = "Welcome to Staff Information System\n" \ "Enter a keyword to start. For help, enter \"help\"." # Object of DataValidator, for validating data self._vld = DataValidator() # Instance of StaffData self._std = StaffData() def do_select(self, line): """ Select a data source :param line: <String> Source name :return: None :Author: Zhiming Liu """ cmd = SelectCommand(line) if cmd.resource_type is None: View.error("The data resource is not available.") elif cmd.resource_type == "csv": self.select_csv(cmd) elif cmd.resource_type == "db": self._std.select_source(cmd.resource_type) def select_csv(self, cmd): try: if not cmd.is_valid_command: View.error("Invalid command.") View.help_select() elif cmd.file_name is None: self._std.select_source(cmd.resource_type, cmd.default_csv_file) View.warning("No CSV file path specified. " "A default file \"{0}\" " "will be used.".format(cmd.default_csv_file)) else: self._std.select_source(cmd.resource_type, cmd.file_name, cmd.create_file) except Exception as e: View.error(e) else: View.success("Data source CSV is selected.") def do_add(self, line): """ Add a new entry of data :param line: (String) [EMPID] [Age] [Gender] [Sales] [BMI] [Salary] [Birthday] :return: None :Author: Zhiming Liu """ # Split the input argument to obtain the data raw_data = list(arg.lower() for arg in str(line).split()) try: # Check if input data has 7 data fields if not len(raw_data) == len(Data): raise AttributeError("Please input correct data.") else: # Check and wash data by check_all() of DataValidator result = self._vld.check_all(raw_data) # Check if there is any None which stands for invalid input if None in result: key = 0 # build a list of name list items = list(map(lambda i: i.name, Data)) e_str = "" while key < len(result): if result[key] is None: # Left alignment e_str += "{:<10}".format(items[key]) key += 1 raise ValueError("The following field(s) " "is invalid:\n%s" % e_str) else: self._std.add_data(result) except (AttributeError, ValueError) as e: View.error(str(e) + "\n") View.help_add() except CSVError as e: View.error(e) except Exception as e: View.error(e) else: View.success("Add data") def do_import(self, line): """ Import command :param line: (String) [-csv|-pk] [file path] :return: None :Author: Zhiming Liu """ cmd = ImportCommand(line) if cmd.file_type is None or cmd.file_name is None: View.info("Invalid command.") View.help_import() elif cmd.file_type == "csv": self.import_csv(cmd.file_name) elif cmd.file_type == "pk": self.import_pickle(cmd.file_name) def import_csv(self, filename): try: csv = CSVOperations(filename) self.import_result(csv.read()) except Exception as e: View.error(e) def import_pickle(self, filename): try: pk = PickleOperations() self.import_result(list(pk.pickle_import(filename))) except Exception as e: View.error(e) def import_result(self, data): View.display_import_result( list(map(lambda row: self.import_row(row), data))) def import_row(self, raw_data): """ Add a new data from imported files :param raw_data: Dict :return: Dict :Author: Zhiming Liu """ result = list(raw_data.values()) try: washed = self._vld.check_all(list(raw_data.values())) if None in washed: result.append("Fail") else: self._std.add_data(washed) result.append("Pass") except Exception: result.append("Fail") finally: return result def do_export(self, line): """ Export command :param line: (String) [-pk] [file path] :return: None :Author: Zhiming Liu """ args = list(arg.lower() for arg in str(line).split()) if args[0] == "-pk" and len(args) > 1: try: pk = PickleOperations() pk.pickle_dump(args[1], self._std.get_all_data()) except Exception as e: View.error(e) View.help_export() else: View.success("Data has been saved to %s" % args[1]) else: View.info("Invalid command") View.help_export() def do_save(self, arg): """ Save data to specified data source :param arg: arg :return: None :Author: Zhiming Liu """ # If no data source selected, prompt user to do so. try: self._std.save_data() except ValueError as e: View.info(e) except (OSError, AttributeError) as e: View.error(e) except Exception as e: View.error(e) else: View.success("Data is saved") def do_show(self, line): """ Show command :param line: (String) [-t|-p|-b] [object] :return: None :Author: Zhiming Liu """ # Get all instructions args = list(arg.lower() for arg in str(line).split()) # Those commands are required single arguments # single_commands = ["-a"] # Those commands are required two arguments plot_commands = ["-p", "-b"] # Show data table if args[0] == "-t": if len(self._std.data) == 0 and len(self._std.new_data) == 0: View.info("No data to display.") if not len(self._std.data) == 0: View.display("ORIGINAL DATA:") View.display_data(self._std.data, ind=True) if not len(self._std.new_data) == 0: View.display("\nNEW DATA:") View.display_data(self._std.new_data, ind=True) View.display("\n(Input command \"save\" to save the new data)") elif args[0] in plot_commands: try: if len(args) == 1: raise IndexError("Incomplete command line.") if args[0] == "-p": self.show_pie(args[1]) if args[0] == "-b": self.show_bar(args[1]) except IndexError as e: View.error(str(e) + "\n") View.help_show() else: View.info("Invalid command line.\n") View.help_show() def show_pie(self, line): """ Draw pie chart :param line: String :return: None :Author: Zhiming Liu """ # Draw Pies try: if self._std.get_gender().total_count == 0 \ or len(self._std.get_bmi()) == 0: raise ValueError("No data to display.") # Draw gender if line.upper() == Data.GENDER.name: View.plot_pie(self._std.get_gender().formatted_data, "Gender Distribution", "People") # Draw BMI if line.upper() == Data.BMI.name: View.plot_pie(self._std.get_bmi(), "Body Mass Index (BMI)", "People") except ValueError as e: View.info(e) except Exception as e: View.error(e) def show_bar(self, line): """ Draw bar chart :param line: String :return: None :Author: Zhiming Liu """ # Draw Bars try: if self._std.get_gender().total_count == 0 \ or len(self._std.get_bmi()) == 0: raise ValueError("No data to display.") # Draw gender if line.upper() == Data.GENDER.name: View.plot_bar(self._std.get_gender().formatted_data, "Gender Distribution", "numer of people") # Draw BMI if line.upper() == Data.BMI.name: View.plot_bar(self._std.get_bmi(), "Body Mass Index (BMI)", "number of people") except ValueError as e: View.info(e) except Exception as e: View.error(e) @staticmethod def help_show(): View.display("This command is used for displaying all data that " "is existed in the system.\n") View.help_show() @staticmethod def help_add(): View.display("This command adds a new staff data to the system.\n") View.help_add() @staticmethod def help_save(): View.display("This command is used for saving all newly added data " "to the selected data source.\n") View.help_save() @staticmethod def help_select(): View.display("Select a source of data for reading " "and saving staff information.\n") View.help_select() @staticmethod def help_quit(): View.display("This command is used for " "quitting the command line mode\n") View.help_quit() @staticmethod def help_import(): View.display("Import data from a file " "that is specified in the command line.\n") View.help_import() @staticmethod def help_export(): View.display("Export data to a local file " "that is specified in the command line.\n") View.help_export() def do_quit(self, line): arg = str(line).lower() if not arg == "-f" and not len(self._std.new_data) == 0: View.warning("The new data hasn't been saved. " "Enter \"quit -f\" to quit without saving.") else: View.display("Thanks for using. Bye!") return True