コード例 #1
0
def DumpTables(dbspec, num_readers, dump_dir, mysql_bin_dir, lossy_fp):
  dbs = dbspec.getDatabases()
  db_info = {}
  for db in dbs:
    db_info[db] = {}
    for table in dbspec.getTables(db):
      db_info[db][table.name] = table

  todo = []
  for db in db_info:
    for table in db_info[db].values():
      if table.table_type == 'InnoDB':
        todo.append((db, table))

  logging.info("Found %d tables to compact" % len(todo))

  # sort descending, first by size then by name
  todo.sort(lambda a,b: cmp(b[1].size, a[1].size) or cmp(b[1].name, a[1].name))

  pool = command_pool.CommandPool(num_readers)
  for db, table in todo:
    # we use PIPESTATUS to get the return of mysqldump, not gzip.
    # also, --lossless-fp is an extension to our mysqldump that allows
    # for truly lossless dumps and reloads.  this indirectly ensures
    # we have the proper mysql in place for the dump, as the flag to
    # mysqldump also requires a mysqld change that is accompanied in
    # the same RPMs.
    if len(mysql_bin_dir):
      cmdpath = "%s/mysqldump" % mysql_bin_dir
    else:
      cmdpath = "mysqldump"
    if lossy_fp:
      lossy = ''
    else:
      lossy = '--lossless_fp'
    cmd = (("%s --opt -v -u%s -p%s -h%s %s --database %s --tables %s | "
            "gzip --fast > %s/%s-%s.sql.gz; exit ${PIPESTATUS[0]}") %
           (cmdpath, dbspec.user, dbspec.password,
            dbspec.host, lossy, db, table.name, dump_dir, db, table.name))
    pool.submit(cmd, (db, table))

  if not pool.run():
    for failure in pool.failures:
      logging.error("Dump of %s.%s failed: %s" %
                    (failure.data[0], failure.data[1], failure.returncode))
      logging.error("Output: %s\n" % failure.output.read())
    logging.fatal("One or more dumps failed; aborting.")
コード例 #2
0
def ReloadTables(dbspec, num_writers, dump_dir, mysql_bin_dir, wipe_innodb):
  todo = GetTableNamesFromDumpFiles(dump_dir)

  if wipe_innodb:
    logging.info("Dropping innodb tables...")
    for db, table in todo:
      logging.debug('Drop %s.%s' % (db, table))
      dbspec.execute("DROP TABLE %s.%s" % (db, table))

    logging.info("Stopping mysql...")
    dbspec.stopMySql()

    logging.info("Deleting innodb data files...")
    retcode = subprocess.call("sudo rm -f %s/innodb_data*; "
                              "sudo rm -f %s/innodb_logs/*"
                              % (FLAGS.mysql_root, FLAGS.mysql_root),
                              shell=True)

    if retcode != 0:
      logging.fatal("Error deleting innodb datafiles")

    logging.info("Starting mysql...")
    dbspec.startMySql()
    time.sleep(5)

  logging.info("Restoring tables...")
  pool = command_pool.CommandPool(num_writers)
  if len(mysql_bin_dir):
    cmdpath = "%s/mysql" % mysql_bin_dir
  else:
    cmdpath = "mysql"

  for db, table in todo:
    cmd = (("zcat %s/%s-%s.sql.gz | %s -u%s -p%s -h%s -A %s") %
           (dump_dir, db, table, cmdpath,
            dbspec.user, dbspec.password, dbspec.host, db,
            ))
    logging.debug('Submit %s' % cmd)
    pool.submit(cmd, (db, table))

  if not pool.run():
    for failure in pool.failures:
      logging.error("Restore of %s.%s failed: %s" %
                    (failure.data[0], failure.data[1], failure.returncode))
      logging.error("Output: %s\n" % failure.output.read())
    logging.fatal("One or more restores failed; aborting.")
コード例 #3
0
def main(argv):
  if len(argv) != 2:
    flags.ShowUsage()
    sys.exit(1)

  dbspec = dbspec_lib.DatabaseSpec(argv[1], FLAGS.user, FLAGS.password)

  if not os.path.exists(FLAGS.dump_dir):
    logging.fatal("Specified --dump_dir, %s, does not exist" % FLAGS.dump_dir)
  if not os.path.isdir(FLAGS.dump_dir):
    logging.fatal("Specified --dump_dir, %s, is not a directory" %
                  FLAGS.dump_dir)
  if os.listdir(FLAGS.dump_dir) and FLAGS.do_dump:
    logging.fatal("Specified --dump_dir, %s, is not empty" % FLAGS.dump_dir)

  if _CheckReplicationConfigured(dbspec):
    logging.fatal("Replication must not be configured during a compaction")

  # ensure we are running a mysql that properly lets us dump and
  # restore floating point numbers
  rows = dbspec.execute("SELECT IEEE754_TO_STRING(1.000)", failOnError=0)
  if not rows:
    if not FLAGS.tolerate_lossy_fp:
      logging.fatal("Your version of mysql does not support precise "
                    "IEEE754 string representations.  Dump aborted.")
    else:
      logging.error("Your version of mysql does not support precise "
                    "IEEE754 string representations.")
  else:
    logging.info("Your mysql supports IEEE754_TO_STRING, dump proceding")

  if FLAGS.do_dump:
    DumpTables(dbspec, FLAGS.num_readers, FLAGS.dump_dir, FLAGS.mysql_bin_dir,
               FLAGS.tolerate_lossy_fp)

  if FLAGS.do_reload:
    ReloadTables(dbspec, FLAGS.num_writers, FLAGS.dump_dir, FLAGS.mysql_bin_dir,
                 FLAGS.wipe_innodb)

  logging.info("Done!")