def get_diff(self, db_name, version): file_schema = "%s/%04d-schema.img" % (db_name, int(version)) file_schema_swp = "%s/%04d-schema.swp" % (db_name, int(version)) tmp_db = "tmp_online_mig_%s" % (db_name) query = "CREATE DATABASE %s;" % tmp_db self.server.disable_foreign_key_checks() self.server.exec_query(query) f = open(file_schema, 'r') f_swp = open(file_schema_swp, 'w') f_swp.write("USE %s\n" % tmp_db) buff = "" for line in f.readlines(): #if re.search(';$', line, re.IGNORECASE): if re.search(';$' + '(?i)', line): buff = buff + line f_swp.write(buff) buff = "" else: buff = buff + line.strip() f.close() f_swp.close() query_options = {'multi': True} f_swp = open(file_schema_swp, 'r') for line in f_swp.readlines(): self.server.exec_query(line, query_options) f_swp.close() os.remove(file_schema_swp) query_options = { 'run_all_tests': True, 'reverse': False, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'differ', 'width': 75, 'changes-for': 'server1', 'skip_grants': True} source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcompare.database_compare(source_values, destination_values, db_name, tmp_db, query_options) buf = "" found = 0 for line in stepback.getvalue().splitlines(True): logging.debug(u"%s" % line) if re.search('CREATE TABLE', line): line = re.sub("CREATE" + '(?i)',"", line, 1) #line = re.sub("CREATE","", line, 1, re.IGNORECASE) #line = re.sub("\(\n","", line, 1, re.IGNORECASE) line = re.sub("\(\n" + '(?i)',"", line, 1) buf=line if (re.search('^\+', line) or re.search('^\-', line)) and not re.search('CREATE DATABASE', line) and not re.search('CONSTRAINT.*FOREIGN KEY', line): #line = re.sub("\n","", line, 1, re.IGNORECASE) line = re.sub("\n" + '(?i)',"", line, 1) if len(buf) > 0: print u"%s" % buf buf="" print u"%s" % line query = "DROP DATABASE %s" % tmp_db self.server.exec_query(query) self.server.disable_foreign_key_checks(disable=False)
def run(self): self.mask_global = False self.res_fname = "result.txt" s1_conn_str = self.build_connection_string(self.server1) s2_conn_str = self.build_connection_string(self.server2) from_conn = "--server={0}".format(s1_conn_str) to_conn = "--server={0}".format(s2_conn_str) cmp_options = { "no_checksum_table": False, "run_all_tests": True, "quiet": True } # There is a bug with the rest of the formats, after fixed we should # enable them. _FORMATS = ("SQL", "CSV", "TAB", "GRID", "VERTICAL") _DISPLAYS = ("BRIEF", "FULL") # We will do "NAMES" in import_errors test_num = 1 res = True for display in _DISPLAYS: for frmt in _FORMATS: # Run import test for each database in the list. for db in self.database_list: comment = ("Test Case {0} : Testing import with {1} " "format and {2} display for database " "'{3}'.").format(test_num, frmt, display, db) if self.verbose: print("\nEXECUTING {0}".format(comment)) # We test DEFINITIONS and DATA only in other tests self.run_import_test(0, from_conn, to_conn, [db], frmt, "BOTH", comment, " --display={0}".format(display)) old_stdout = sys.stdout try: # redirect stdout to prevent database_compare prints # to reach the MUT output. if not self.debug: sys.stdout = open(os.devnull, 'w') # Test correctness of data res = database_compare(s1_conn_str, s2_conn_str, db, db, cmp_options) if not res: break finally: # restore stdout if not self.debug: sys.stdout.close() sys.stdout = old_stdout test_num += 1 # Drop databases to import them again from file self.server2.disable_foreign_key_checks(True) for db in self.database_list: self.drop_db(self.server2, db, self.debug) if not res: raise MUTLibError("{0} failed".format(comment)) self.server2.disable_foreign_key_checks(False) return True
def run(self): self.mask_global = False self.res_fname = "result.txt" s1_conn_str = self.build_connection_string(self.server1) s2_conn_str = self.build_connection_string(self.server2) from_conn = "--server={0}".format(s1_conn_str) to_conn = "--server={0}".format(s2_conn_str) cmp_options = {"no_checksum_table": False, "run_all_tests": True, "quiet": True} # There is a bug with the rest of the formats, after fixed we should # enable them. _FORMATS = ("SQL", "CSV", "TAB", "GRID", "VERTICAL") _DISPLAYS = ("BRIEF", "FULL") # We will do "NAMES" in import_errors test_num = 1 res = True for display in _DISPLAYS: for frmt in _FORMATS: # Run import test for each database in the list. for db in self.database_list: comment = ("Test Case {0} : Testing import with {1} " "format and {2} display for database " "'{3}'.").format(test_num, frmt, display, db) if self.verbose: print("\nEXECUTING {0}".format(comment)) # We test DEFINITIONS and DATA only in other tests self.run_import_test( 0, from_conn, to_conn, [db], frmt, "BOTH", comment, " --display={0}".format(display) ) old_stdout = sys.stdout try: # redirect stdout to prevent database_compare prints # to reach the MUT output. if not self.debug: sys.stdout = open(os.devnull, 'w') # Test correctness of data res = database_compare(s1_conn_str, s2_conn_str, db, db, cmp_options) if not res: break finally: # restore stdout if not self.debug: sys.stdout.close() sys.stdout = old_stdout test_num += 1 # Drop databases to import them again from file self.server2.disable_foreign_key_checks(True) for db in self.database_list: self.drop_db(self.server2, db, self.debug) if not res: raise MUTLibError("{0} failed".format(comment)) self.server2.disable_foreign_key_checks(False) return True
parser.error(PARSE_ERR_DB_PAIR_EXT.format(db_pair=db, db1_label='db1', db2_label='db2', db1_value=parts[0], db2_value=parts[1])) # Remove backtick quotes (handled later) db1 = remove_backtick_quoting(parts[0], server1_sql_mode) \ if is_quoted_with_backticks(parts[0], server1_sql_mode) \ else parts[0] db2 = remove_backtick_quoting(parts[1], server1_sql_mode) \ if is_quoted_with_backticks(parts[1], server1_sql_mode) \ else parts[1] try: res = database_compare(server1_values, server2_values, db1, db2, options) if not opt.quiet: print except UtilError: _, e, _ = sys.exc_info() print("ERROR: %s" % e.errmsg) sys.exit(1) if not res: check_failed = True if check_failed and not opt.run_all_tests: break if not opt.quiet: print
def migrate_up(self, db_name, last_version): (ver, md5, comment) = self.read_meta(db_name, int(last_version)) if self.verify_checksum(db_name, last_version, md5) is False: logging.warning( "The current schema doesn't match the last applied migration") version = self.new_migration_version(db_name) if not os.path.exists("%s/%04d-up.mig" % (db_name, int(version))): logging.info("No migration available") else: logging.info(u"Preparing migration to version %04d" % int(version)) if os.path.exists("%s/%04d-down.mig" % (db_name, int(version))): os.remove("%s/%04d-down.mig" % (db_name, int(version))) (ver, md5, comment) = self.read_meta(db_name, int(version)) query_options = {'skip_data': True, 'force': True} db_list = [] grp = re.match("(\w+)(?:\:(\w+))?", "%s:%s_%s" % (db_name, self.tmp_prefix, db_name)) db_entry = grp.groups() db_list.append(db_entry) source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcopy.copy_db(source_values, destination_values, db_list, query_options) self.online_schema_change( db_name, version, "%s/%04d-up.mig" % (db_name, int(version))) if self.verify_checksum(db_name, version, md5) is True: logging.info("Applied changes match the requested schema") else: logging.error( "Something didn't run as expected, db schema doesn't match !" ) self.change_migration_status(db_name, version, 'invalid checksum') query_options = { 'run_all_tests': True, 'reverse': True, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'sql', 'width': 75, 'changes-for': 'server1', 'skip_grants': True, 'skip_gtid': True } with capture() as stepback: res = dbcompare.database_compare( source_values, destination_values, db_name, "%s_%s" % (self.tmp_prefix, db_name), query_options) str = stepback.getvalue().splitlines(True) to_add = 0 file_down = open("%s/%04d-down.tmp" % (db_name, int(version)), 'a') for line in str: if line[0] not in ['#', '\n', '+', '-', '@']: # this if is required currently due to missing foreign keys in dbcopy if not re.match("\s+DROP FOREIGN KEY", line): #line = re.sub(" %s\." % db_name, " ", line, 1, re.IGNORECASE) line = re.sub(" %s\." % db_name + '(?i)', " ", line, 1) file_down.write("%s\n" % line.strip()) elif re.match("# WARNING: Objects in", line): if re.match("# WARNING: Objects in \w+\.tmp_online_mig_", line): to_add = 2 else: to_add = 1 else: grp = re.match("#\s+TABLE\: (\w+)", line) if grp: if to_add == 2: query = "SHOW CREATE TABLE tmp_online_mig_%s.%s;" % ( db_name, grp.group(1)) res = self.server.exec_query(query) file_down.write("%s\n" % res[0][1]) elif to_add == 1: file_down.write("DROP TABLE %s;\n" % grp.group(1)) file_down.close() file_down_tmp = "%s/%04d-down.tmp" % (db_name, int(version)) self.create_migration_file(db_name, file_down_tmp, version, "down") query = "DROP DATABASE %s_%s" % (self.tmp_prefix, db_name) res = self.server.exec_query(query) #os.remove(file_down_tmp) file_schema = "%s/%04d-schema.img" % (db_name, int(version)) self.create_schema_img(db_name, file_schema)
def get_diff(self, db_name, version): file_schema = "%s/%04d-schema.img" % (db_name, int(version)) file_schema_swp = "%s/%04d-schema.swp" % (db_name, int(version)) tmp_db = "tmp_online_mig_%s" % (db_name) query = "CREATE DATABASE %s;" % tmp_db self.server.disable_foreign_key_checks() self.server.exec_query(query) f = open(file_schema, 'r') f_swp = open(file_schema_swp, 'w') f_swp.write("USE %s\n" % tmp_db) buff = "" for line in f.readlines(): #if re.search(';$', line, re.IGNORECASE): if re.search(';$' + '(?i)', line): buff = buff + line f_swp.write(buff) buff = "" else: buff = buff + line.strip() f.close() f_swp.close() query_options = {'multi': True} f_swp = open(file_schema_swp, 'r') for line in f_swp.readlines(): self.server.exec_query(line, query_options) f_swp.close() os.remove(file_schema_swp) query_options = { 'run_all_tests': True, 'reverse': False, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'differ', 'width': 75, 'changes-for': 'server1', 'skip_grants': True, 'skip_gtid': True } source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcompare.database_compare(source_values, destination_values, db_name, tmp_db, query_options) buf = "" found = 0 for line in stepback.getvalue().splitlines(True): logging.debug(u"%s" % line) if re.search('CREATE TABLE', line): line = re.sub("CREATE" + '(?i)', "", line, 1) #line = re.sub("CREATE","", line, 1, re.IGNORECASE) #line = re.sub("\(\n","", line, 1, re.IGNORECASE) line = re.sub("\(\n" + '(?i)', "", line, 1) buf = line if (re.search('^\+', line) or re.search('^\-', line)) and not re.search( 'CREATE DATABASE', line) and not re.search( 'CONSTRAINT.*FOREIGN KEY', line): #line = re.sub("\n","", line, 1, re.IGNORECASE) line = re.sub("\n" + '(?i)', "", line, 1) if len(buf) > 0: print u"%s" % buf buf = "" print u"%s" % line query = "DROP DATABASE %s" % tmp_db self.server.exec_query(query) self.server.disable_foreign_key_checks(disable=False)
# 1) databases exist # 2) check object counts # 3) check object differences # 4) check row counts among the tables # 5) check table data consistency res = True check_failed = False for db in args: parts = db.split(":") if len(parts) == 1: parts.append(parts[0]) elif len(parts) != 2: parser.error("Invalid format for database compare argument. " "Format should be: db1:db2 or db.") try: res = database_compare(server1_values, server2_values, parts[0], parts[1], options) print except UtilError, e: print "ERROR:", e.errmsg check_failed = True if not opt.run_all_tests: break if not res: check_failed = True if check_failed and not opt.run_all_tests: break if not opt.quiet: print if check_failed:
def migrate_up(self, db_name, last_version): (ver, md5, comment) = self.read_meta(db_name, int(last_version)) if self.verify_checksum(db_name, last_version, md5) is False: logging.warning("The current schema doesn't match the last applied migration") version = self.new_migration_version(db_name) if not os.path.exists("%s/%04d-up.mig" % (db_name, int(version))): logging.info("No migration available") else: logging.info(u"Preparing migration to version %04d" % int(version)) if os.path.exists("%s/%04d-down.mig" % (db_name, int(version))): os.remove("%s/%04d-down.mig" % (db_name, int(version))) (ver, md5, comment) = self.read_meta(db_name, int(version)) query_options = {'skip_data': True, 'force': True} db_list = [] grp = re.match("(\w+)(?:\:(\w+))?", "%s:%s_%s" % (db_name, self.tmp_prefix, db_name)) db_entry = grp.groups() db_list.append(db_entry) source_values = parse_connection(server_connection) destination_values = parse_connection(server_connection) with capture() as stepback: dbcopy.copy_db(source_values, destination_values, db_list, query_options) self.online_schema_change(db_name, version, "%s/%04d-up.mig" % (db_name, int(version))) if self.verify_checksum(db_name, version, md5) is True: logging.info("Applied changes match the requested schema") else: logging.error("Something didn't run as expected, db schema doesn't match !") self.change_migration_status(db_name, version, 'invalid checksum') query_options = { 'run_all_tests': True, 'reverse': True, 'verbosity': None, 'no_object_check': False, 'no_data': True, 'quiet': True, 'difftype': 'sql', 'width': 75, 'changes-for': 'server1', 'skip_grants': True} with capture() as stepback: res = dbcompare.database_compare(source_values, destination_values, db_name, "%s_%s" % (self.tmp_prefix, db_name), query_options) str = stepback.getvalue().splitlines(True) to_add = 0 file_down = open("%s/%04d-down.tmp" % (db_name, int(version)), 'a') for line in str: if line[0] not in ['#', '\n', '+', '-', '@']: # this if is required currently due to missing foreign keys in dbcopy if not re.match("\s+DROP FOREIGN KEY", line): #line = re.sub(" %s\." % db_name, " ", line, 1, re.IGNORECASE) line = re.sub(" %s\." % db_name + '(?i)', " ", line, 1) file_down.write("%s\n" % line.strip()) elif re.match("# WARNING: Objects in", line): if re.match("# WARNING: Objects in \w+\.tmp_online_mig_", line): to_add = 2 else: to_add = 1 else: grp = re.match("#\s+TABLE\: (\w+)", line) if grp: if to_add == 2: query = "SHOW CREATE TABLE tmp_online_mig_%s.%s;" % (db_name, grp.group(1)) res = self.server.exec_query(query) file_down.write("%s\n" % res[0][1]) elif to_add == 1: file_down.write("DROP TABLE %s;\n" % grp.group(1)) file_down.close() file_down_tmp = "%s/%04d-down.tmp" % (db_name, int(version)) self.create_migration_file(db_name, file_down_tmp, version, "down") query = "DROP DATABASE %s_%s" % (self.tmp_prefix, db_name) res = self.server.exec_query(query) #os.remove(file_down_tmp) file_schema = "%s/%04d-schema.img" % (db_name, int(version)) self.create_schema_img(db_name, file_schema)
def run(self): self.res_fname = "result.txt" s1_conn_str = self.build_connection_string(self.server1) s2_conn_str = self.build_connection_string(self.server2) from_conn = "--server={0}".format(s1_conn_str) to_conn = "--server={0}".format(s2_conn_str) cmp_options = {"no_checksum_table": False, "run_all_tests": True, "quiet": True} cmd = ("mysqldbimport.py {0} --import=definitions " "{1}").format(to_conn, self.export_import_file) case_num = 1 comment = "Test case {0} - help".format(case_num) cmd_opts = " --help" cmd_str = "{0} {1}".format(cmd, cmd_opts) res = self.run_test_case(0, cmd_str, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) # Remove version information self.remove_result_and_lines_after("MySQL Utilities mysqldbimport.py " "version", 6) # Now test the skips # Note: data and blobs must be done separately _SKIPS = ("grants", "events", "triggers", "views", "procedures", "functions", "tables", "create_db") _FORMATS = ("CSV", "SQL") case_num += 1 for frmt in _FORMATS: # Create an import file export_cmd = ("mysqldbexport.py {0} util_test --export=BOTH " "--skip-gtid --format={1} --display=BRIEF > " "{2}").format(from_conn, frmt, self.export_import_file) comment = "Generating import file" res = self.run_test_case(0, export_cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) cmd_opts = "{0} --format={1} --skip=".format(cmd, frmt) for skip in _SKIPS: if case_num != 2 and case_num != 2 + len(_SKIPS): cmd_opts = "{0},".format(cmd_opts) cmd_opts = "{0}{1}".format(cmd_opts, skip) comment = "Test case {0} - no {1}".format(case_num, skip) self.do_skip_test(cmd_opts, comment) case_num += 1 # Now test --skip=data, --skip-blobs # Create an import file with blobs try: self.server1.exec_query("ALTER TABLE util_test.t3 " "ADD COLUMN me_blob BLOB") self.server1.exec_query("UPDATE util_test.t3 SET " "me_blob = 'This, is a BLOB!'") except UtilDBError as err: raise MUTLibError("Failed to add blob column: " "{0}".format(err.errmsg)) export_cmd = ("mysqldbexport.py {0} util_test --export=BOTH " "--skip-gtid --format={1} --display=BRIEF > " "{2} ").format(from_conn, "CSV", self.export_import_file) comment = "Generating import file" res = self.run_test_case(0, export_cmd, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) # No skips for reference (must skip events for deterministic reasons cmd_str = ("mysqldbimport.py {0} {1} --import=both --dryrun " "--format=CSV --bulk-insert " "--skip=events").format(to_conn, self.export_import_file) comment = "Test case {0} - no {1}".format(case_num, "events") res = self.run_test_case(0, cmd_str, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) case_num += 1 cmd_str = ("mysqldbimport.py {0} {1} --import=both --dryrun " "--format=CSV --bulk-insert " "--skip=events," "data".format(to_conn, self.export_import_file)) comment = "Test case {0} - no {1}".format(case_num, "data") res = self.run_test_case(0, cmd_str, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) case_num += 1 cmd_str = ("mysqldbimport.py {0} {1} --import=both --dryrun " "--format=CSV --skip-blobs --bulk-insert " "--skip=events").format(to_conn, self.export_import_file) comment = "Test case {0} - no {1}".format(case_num, "blobs") res = self.run_test_case(0, cmd_str, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) # Do a quiet import case_num += 1 cmd_str = ("mysqldbimport.py {0} {1} --import=both --quiet " "--format=CSV " "--bulk-insert").format(to_conn, self.export_import_file) comment = "Test case {0} - no {1}".format(case_num, "messages (quiet)") res = self.run_test_case(0, cmd_str, comment) if not res: raise MUTLibError("{0}: failed".format(comment)) # Import using multiprocessing. case_num += 1 comment = "Test case {0} - multiprocessing.".format(case_num) import_opts = "--multiprocess=2" self.drop_db(self.server2, 'util_test') # drop db before import. self.run_import_test(0, from_conn, to_conn, ['util_test'], "SQL", "BOTH", comment, "", import_opts) # Import using autocommit. case_num += 1 comment = "Test case {0} - autocommit.".format(case_num) import_opts = "--autocommit" self.drop_db(self.server2, 'util_test') # drop db before import. self.run_import_test(0, from_conn, to_conn, ['util_test'], "SQL", "BOTH", comment, "", import_opts) # Import multiple files at once # Test multiple formats and displays _FORMATS = ("SQL", "CSV", "TAB", "GRID", "VERTICAL") _DISPLAYS = ("BRIEF", "FULL") case_num += 1 res = True database_list = ['util_test_fk', 'util_test_fk2', 'util_test_fk3'] # Drop existing databases on both servers and load databases from # fkeys.sql into both server1 and server2, since they do not have # unsupported features (for formats other than sql) such as # auto-increment. self.server1.disable_foreign_key_checks(True) self.server2.disable_foreign_key_checks(True) self.drop_all() for db in database_list: self.drop_db(self.server1, db) self.drop_db(self.server2, db) # Load databases from fkeys.sql into both server1 and server2, since # they do not have unsupported csv features such as auto-increment. data_file = os.path.normpath("./std_data/fkeys.sql") try: self.server1.read_and_exec_SQL(data_file, self.debug) self.server2.read_and_exec_SQL(data_file, self.debug) except UtilError as err: raise MUTLibError("Failed to read commands from file " "{0}: {1}".format(data_file, err.errmsg)) self.server1.disable_foreign_key_checks(False) self.server2.disable_foreign_key_checks(False) for frmt in _FORMATS: for display in _DISPLAYS: comment = ("Test Case {0} : Testing multiple import with {1} " "format and {2} display".format(case_num, frmt, display)) # We test DEFINITIONS and DATA separately in other tests self.run_import_test( 0, from_conn, to_conn, database_list, frmt, "BOTH", comment, " --display={0}".format(display) ) old_stdout = sys.stdout try: # redirect stdout to prevent database_compare prints # to reach the MUT output. if not self.verbose: sys.stdout = open(os.devnull, 'w') # Test correctness of data for database in database_list: res = database_compare(s1_conn_str, s2_conn_str, database, database, cmp_options) if not res: break finally: # restore stdout if not self.verbose: sys.stdout.close() sys.stdout = old_stdout # Drop dbs from server2 to import them again from the # export file self.server2.disable_foreign_key_checks(True) for db in database_list: self.drop_db(self.server2, db) self.server2.disable_foreign_key_checks(False) if not res: raise MUTLibError("{0} failed".format(comment)) case_num += 1 # Mask multiprocessing warning. self.remove_result("# WARNING: Number of processes ") # Mask version self.replace_result( "MySQL Utilities mysqldbimport version", "MySQL Utilities mysqldbimport version X.Y.Z\n") return res
# of backticks. if matched_size != len(db): parser.error(PARSE_ERR_DB_PAIR_EXT.format(db_pair=db, db1_label='db1', db2_label='db2', db1_value=parts[0], db2_value=parts[1])) # Remove backtick quotes (handled later) db1 = remove_backtick_quoting(parts[0]) \ if is_quoted_with_backticks(parts[0]) else parts[0] db2 = remove_backtick_quoting(parts[1]) \ if is_quoted_with_backticks(parts[1]) else parts[1] try: res = database_compare(server1_values, server2_values, db1, db2, options) print except UtilError: _, e, _ = sys.exc_info() print("ERROR: %s" % e.errmsg) check_failed = True if not opt.run_all_tests: break if not res: check_failed = True if check_failed and not opt.run_all_tests: break if not opt.quiet: print
# 2) check object counts # 3) check object differences # 4) check row counts among the tables # 5) check table data consistency res = True check_failed = False for db in args: parts = db.split(":") if len(parts) == 1: parts.append(parts[0]) elif len(parts) != 2: parser.error("Invalid format for database compare argument. " "Format should be: db1:db2 or db.") try: res = database_compare(server1_values, server2_values, parts[0], parts[1], options) print except UtilError, e: print "ERROR:", e.errmsg check_failed = True if not opt.run_all_tests: break if not res: check_failed = True if check_failed and not opt.run_all_tests: break if not opt.quiet: print if check_failed: