def compare_metadata(self, start_time, tables):
prod_connection = dbcmp_sql_helper.DbCmpSqlHelper(
self.prod_sql, self.logger)
test_connection = dbcmp_sql_helper.DbCmpSqlHelper(
self.test_sql, self.logger)
for table in tables:
self.logger.info("Check schema for table {}...".format(table))
query = (
"SELECT {} FROM INFORMATION_SCHEMA.COLUMNS ".format(', '.join(
self.schema_columns)) +
"WHERE TABLE_SCHEMA = 'DBNAME' AND TABLE_NAME='TABLENAME' ".
replace("TABLENAME", table) + "ORDER BY COLUMN_NAME;")
prod_columns, test_columns = dbcmp_sql_helper.get_comparable_objects(
[prod_connection, test_connection], query)
if (prod_columns is None) or (test_columns is None):
self.logger.warn(
'Table {} skipped because something going bad'.format(
table))
continue
# TODO: Type error: unhashable type: 'dict'
uniq_for_prod = list(set(prod_columns) - set(test_columns))
uniq_for_test = list(set(test_columns) - set(prod_columns))
if len(uniq_for_prod) > 0 and self.fail_with_first_error:
return self.schema_comparing_time(table, uniq_for_prod,
start_time)
if len(uniq_for_test) > 0 and self.fail_with_first_error:
return self.schema_comparing_time(table, uniq_for_test,
start_time)
if not all([len(uniq_for_prod) == 0, len(uniq_for_test) == 0]):
self.logger.error(" [ERROR] Tables {} differs!".format(table))
schema_comparing_time = datetime.datetime.now() - start_time
self.logger.info("Schema compared in {}".format(schema_comparing_time))
return datetime.datetime.now() - start_time
def check_test_connection(self):
states = self.statusBar.currentMessage().split(', ')
prod_state = states[0]
test_state = states[1]
if all([
self.le_test_host.text(),
self.le_test_user.text(),
self.le_test_password.text(),
self.le_test_db.text()
]):
test_dict = {
'host': self.le_test_host.text(),
'user': self.le_test_user.text(),
'password': self.le_test_password.text(),
'db': self.le_test_db.text()
}
logger = Logger(self.logging_level)
test_state = 'test disconnected'
for attempt in range(1, 3):
kwargs = {'read_timeout': attempt}
try:
self.test_tables = dbcmp_sql_helper.DbCmpSqlHelper(
test_dict, logger, **kwargs).get_tables()
if self.test_tables:
test_state = 'test connected'
except pymysql.InternalError as e:
logger.error('Exception is {}'.format(e.args[1]))
self.statusBar.showMessage('{}, {}'.format(prod_state, test_state))
def compare_data(self, start_time, service_dir, mapping, tables):
prod_connection = dbcmp_sql_helper.DbCmpSqlHelper(
self.prod_sql, self.logger)
test_connection = dbcmp_sql_helper.DbCmpSqlHelper(
self.test_sql, self.logger)
for table in tables:
# table = 'campaignosreport'
start_table_check_time = datetime.datetime.now()
self.logger.info(
"Table {} processing started now...".format(table))
is_report = self.is_report(table, prod_connection)
# TODO: refactor this place! Rename onlyReports/Entities
if 'onlyReports' in self.separate_checking and not is_report:
continue
if 'onlyEntities' in self.separate_checking and is_report:
continue
self.sql_comparing_properties.update({'service_dir': service_dir})
cmp_params = self.sql_comparing_properties
compared_table = Comparation(prod_connection, test_connection,
table, self.logger, cmp_params)
global_break = compared_table.compare_table(
is_report, mapping, start_time, self.comparing_info,
self.sql_comparing_properties.get('comparing_step'))
self.logger.info(
"Table {} ".format(table) +
"checked in {}...".format(datetime.datetime.now() -
start_table_check_time))
if global_break:
data_comparing_time = datetime.datetime.now() - start_time
self.logger.warn(
('Global breaking is True. Comparing interrupted. ' +
'Comparing finished in {}'.format(data_comparing_time)))
return data_comparing_time
data_comparing_time = datetime.datetime.now() - start_time
self.logger.info(
"Comparing finished in {}".format(data_comparing_time))
return data_comparing_time
def check_test(self, disable_mboxes):
logger = Logger(self.logging_level)
empty_fields = []
if not self.le_test_host.text():
empty_fields.append('test.host')
if not self.le_test_user.text():
empty_fields.append('test.user')
if not self.le_test_password.text():
empty_fields.append('test.password')
if not self.le_test_db.text():
empty_fields.append('test.db')
if empty_fields:
if not disable_mboxes:
QMessageBox.question(
PyQt5.QtWidgets.QMessageBox(), 'Error',
"Please, set this parameter:\n\n" +
"\n".join(empty_fields), QMessageBox.Ok, QMessageBox.Ok)
return False
else:
QMessageBox.question(
PyQt5.QtWidgets.QMessageBox(), 'Error',
"Please, set this parameters:\n\n" +
"\n".join(empty_fields), QMessageBox.Ok, QMessageBox.Ok)
return False
test_host_value = self.le_test_host.text()
test_user_value = self.le_test_user.text()
test_password_value = self.le_test_password.text()
test_db_value = self.le_test_db.text()
test_dict = {
'host': test_host_value,
'user': test_user_value,
'password': test_password_value,
'db': test_db_value
}
try:
dbcmp_sql_helper.DbCmpSqlHelper(test_dict, logger).get_tables()
logger.info('Connection to db {} established successfully!'.format(
self.le_test_db.text()))
if not disable_mboxes:
QMessageBox.information(
PyQt5.QtWidgets.QMessageBox(), 'Information',
"Successfully connected to\n {}/{}".format(
test_dict.get('host'),
test_dict.get('db')), QMessageBox.Ok, QMessageBox.Ok)
return True
except pymysql.OperationalError as err:
logger.warn("Connection to {}/{} ".format(test_dict.get('host'),
test_dict.get('db')) +
"failed\n\n{}".format(err.args[1]))
QMessageBox.warning(
PyQt5.QtWidgets.QMessageBox(), 'Warning',
"Connection to {}/{} failed\n\n{}".format(
test_dict.get('host'), test_dict.get('db'),
err.args[1]), QMessageBox.Ok, QMessageBox.Ok)
return False
except pymysql.InternalError as err:
logger.warn("Connection to {}/{} ".format(test_dict.get('host'),
test_dict.get('db')) +
"failed\n\n{}".format(err.args[1]))
QMessageBox.warning(
PyQt5.QtWidgets.QMessageBox(), 'Warning',
"Connection to {}/{} ".format(test_dict.get('host'),
test_dict.get('db')) +
"failed\n\n{}".format(err.args[1]), QMessageBox.Ok,
QMessageBox.Ok)
return False
'only_tables':
config.get_property('sqlProperties', 'separate_checking'),
'reports':
config.get_property('sqlProperties', 'reports'),
'table_timeout':
config.get_property('sqlProperties', 'table_timeout'),
'os':
os_type
# TODO: support other parameters
}
comparing_info = tableData.Info(logger)
comparing_info.update_table_list(
"prod",
dbcmp_sql_helper.DbCmpSqlHelper(
client_config.get_sql_connection_params("prod"),
logger).get_tables())
comparing_info.update_table_list(
"test",
dbcmp_sql_helper.DbCmpSqlHelper(
client_config.get_sql_connection_params("test"),
logger).get_tables())
global_break = False
if "Linux" in os_type:
create_test_dir("/mxf/data/test_results/", client)
else:
create_test_dir("C:\\dbComparator\\", client)
start_time = datetime.datetime.now()
logger.info("Start {} processing!".format(client))
prod_sql_connection = dbcmp_sql_helper.DbCmpSqlHelper(
client_config.get_sql_connection_params("prod"), logger)