def compare_metadata(self, start_time, tables): prod_connection = dbcmp_sql_helper.DbCmpSqlHelper( self.prod_sql, self.logger) test_connection = dbcmp_sql_helper.DbCmpSqlHelper( self.test_sql, self.logger) for table in tables: self.logger.info("Check schema for table {}...".format(table)) query = ( "SELECT {} FROM INFORMATION_SCHEMA.COLUMNS ".format(', '.join( self.schema_columns)) + "WHERE TABLE_SCHEMA = 'DBNAME' AND TABLE_NAME='TABLENAME' ". replace("TABLENAME", table) + "ORDER BY COLUMN_NAME;") prod_columns, test_columns = dbcmp_sql_helper.get_comparable_objects( [prod_connection, test_connection], query) if (prod_columns is None) or (test_columns is None): self.logger.warn( 'Table {} skipped because something going bad'.format( table)) continue # TODO: Type error: unhashable type: 'dict' uniq_for_prod = list(set(prod_columns) - set(test_columns)) uniq_for_test = list(set(test_columns) - set(prod_columns)) if len(uniq_for_prod) > 0 and self.fail_with_first_error: return self.schema_comparing_time(table, uniq_for_prod, start_time) if len(uniq_for_test) > 0 and self.fail_with_first_error: return self.schema_comparing_time(table, uniq_for_test, start_time) if not all([len(uniq_for_prod) == 0, len(uniq_for_test) == 0]): self.logger.error(" [ERROR] Tables {} differs!".format(table)) schema_comparing_time = datetime.datetime.now() - start_time self.logger.info("Schema compared in {}".format(schema_comparing_time)) return datetime.datetime.now() - start_time
def check_test_connection(self): states = self.statusBar.currentMessage().split(', ') prod_state = states[0] test_state = states[1] if all([ self.le_test_host.text(), self.le_test_user.text(), self.le_test_password.text(), self.le_test_db.text() ]): test_dict = { 'host': self.le_test_host.text(), 'user': self.le_test_user.text(), 'password': self.le_test_password.text(), 'db': self.le_test_db.text() } logger = Logger(self.logging_level) test_state = 'test disconnected' for attempt in range(1, 3): kwargs = {'read_timeout': attempt} try: self.test_tables = dbcmp_sql_helper.DbCmpSqlHelper( test_dict, logger, **kwargs).get_tables() if self.test_tables: test_state = 'test connected' except pymysql.InternalError as e: logger.error('Exception is {}'.format(e.args[1])) self.statusBar.showMessage('{}, {}'.format(prod_state, test_state))
def compare_data(self, start_time, service_dir, mapping, tables): prod_connection = dbcmp_sql_helper.DbCmpSqlHelper( self.prod_sql, self.logger) test_connection = dbcmp_sql_helper.DbCmpSqlHelper( self.test_sql, self.logger) for table in tables: # table = 'campaignosreport' start_table_check_time = datetime.datetime.now() self.logger.info( "Table {} processing started now...".format(table)) is_report = self.is_report(table, prod_connection) # TODO: refactor this place! Rename onlyReports/Entities if 'onlyReports' in self.separate_checking and not is_report: continue if 'onlyEntities' in self.separate_checking and is_report: continue self.sql_comparing_properties.update({'service_dir': service_dir}) cmp_params = self.sql_comparing_properties compared_table = Comparation(prod_connection, test_connection, table, self.logger, cmp_params) global_break = compared_table.compare_table( is_report, mapping, start_time, self.comparing_info, self.sql_comparing_properties.get('comparing_step')) self.logger.info( "Table {} ".format(table) + "checked in {}...".format(datetime.datetime.now() - start_table_check_time)) if global_break: data_comparing_time = datetime.datetime.now() - start_time self.logger.warn( ('Global breaking is True. Comparing interrupted. ' + 'Comparing finished in {}'.format(data_comparing_time))) return data_comparing_time data_comparing_time = datetime.datetime.now() - start_time self.logger.info( "Comparing finished in {}".format(data_comparing_time)) return data_comparing_time
def check_test(self, disable_mboxes): logger = Logger(self.logging_level) empty_fields = [] if not self.le_test_host.text(): empty_fields.append('test.host') if not self.le_test_user.text(): empty_fields.append('test.user') if not self.le_test_password.text(): empty_fields.append('test.password') if not self.le_test_db.text(): empty_fields.append('test.db') if empty_fields: if not disable_mboxes: QMessageBox.question( PyQt5.QtWidgets.QMessageBox(), 'Error', "Please, set this parameter:\n\n" + "\n".join(empty_fields), QMessageBox.Ok, QMessageBox.Ok) return False else: QMessageBox.question( PyQt5.QtWidgets.QMessageBox(), 'Error', "Please, set this parameters:\n\n" + "\n".join(empty_fields), QMessageBox.Ok, QMessageBox.Ok) return False test_host_value = self.le_test_host.text() test_user_value = self.le_test_user.text() test_password_value = self.le_test_password.text() test_db_value = self.le_test_db.text() test_dict = { 'host': test_host_value, 'user': test_user_value, 'password': test_password_value, 'db': test_db_value } try: dbcmp_sql_helper.DbCmpSqlHelper(test_dict, logger).get_tables() logger.info('Connection to db {} established successfully!'.format( self.le_test_db.text())) if not disable_mboxes: QMessageBox.information( PyQt5.QtWidgets.QMessageBox(), 'Information', "Successfully connected to\n {}/{}".format( test_dict.get('host'), test_dict.get('db')), QMessageBox.Ok, QMessageBox.Ok) return True except pymysql.OperationalError as err: logger.warn("Connection to {}/{} ".format(test_dict.get('host'), test_dict.get('db')) + "failed\n\n{}".format(err.args[1])) QMessageBox.warning( PyQt5.QtWidgets.QMessageBox(), 'Warning', "Connection to {}/{} failed\n\n{}".format( test_dict.get('host'), test_dict.get('db'), err.args[1]), QMessageBox.Ok, QMessageBox.Ok) return False except pymysql.InternalError as err: logger.warn("Connection to {}/{} ".format(test_dict.get('host'), test_dict.get('db')) + "failed\n\n{}".format(err.args[1])) QMessageBox.warning( PyQt5.QtWidgets.QMessageBox(), 'Warning', "Connection to {}/{} ".format(test_dict.get('host'), test_dict.get('db')) + "failed\n\n{}".format(err.args[1]), QMessageBox.Ok, QMessageBox.Ok) return False
'only_tables': config.get_property('sqlProperties', 'separate_checking'), 'reports': config.get_property('sqlProperties', 'reports'), 'table_timeout': config.get_property('sqlProperties', 'table_timeout'), 'os': os_type # TODO: support other parameters } comparing_info = tableData.Info(logger) comparing_info.update_table_list( "prod", dbcmp_sql_helper.DbCmpSqlHelper( client_config.get_sql_connection_params("prod"), logger).get_tables()) comparing_info.update_table_list( "test", dbcmp_sql_helper.DbCmpSqlHelper( client_config.get_sql_connection_params("test"), logger).get_tables()) global_break = False if "Linux" in os_type: create_test_dir("/mxf/data/test_results/", client) else: create_test_dir("C:\\dbComparator\\", client) start_time = datetime.datetime.now() logger.info("Start {} processing!".format(client)) prod_sql_connection = dbcmp_sql_helper.DbCmpSqlHelper( client_config.get_sql_connection_params("prod"), logger)