Example #1
0
  def setUp(self):
    if self.vttop is None:
      raise EnvironmentError("VTTOP not defined")
    if self.vtroot is None:
      raise EnvironmentError("VTROOT not defined")
    try:
      os.makedirs(environment.tmproot)
    except OSError:
      pass

    environment.setup()

    framework.execute('go install', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/vtocc')
    framework.execute('go install', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')

    # start mysql
    res = subprocess.call([
        self.vtroot+"/bin/mysqlctl",
        "-tablet-uid",  self.tabletuid,
        "-port", str(self.vtoccport),
        "-mysql-port", str(self.mysqlport),
        "init"
        ])
    if res != 0:
      raise EnvironmentError("Cannot start mysql")
    res = subprocess.call([
        "mysql",
        "-S",  self.mysqldir+"/mysql.sock",
        "-u", "vt_dba",
        "-e", "create database vt_test_keyspace ; set global read_only = off"])
    if res != 0:
      raise Exception("Cannot create vt_test_keyspace database")

    self.mysql_conn = self.mysql_connect()
    mcu = self.mysql_conn.cursor()
    self.clean_sqls = []
    self.init_sqls = []
    clean_mode = False
    with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
      for line in f:
        line = line.rstrip()
        if line == "# clean":
          clean_mode = True
        if line=='' or line.startswith("#"):
          continue
        if clean_mode:
          self.clean_sqls.append(line)
        else:
          self.init_sqls.append(line)
    try:
      for line in self.init_sqls:
        mcu.execute(line, {})
    finally:
      mcu.close()

    customrules = '/tmp/customrules.json'
    self.create_customrules(customrules)
    schema_override = '/tmp/schema_override.json'
    self.create_schema_override(schema_override)

    occ_args = [
      self.vtroot+"/bin/vtocc",
      "-port", "9461",
      "-customrules", customrules,
      "-schema-override", schema_override,
      "-db-config-app-charset", "utf8",
      "-db-config-app-dbname", "vt_test_keyspace",
      "-db-config-app-host", "localhost",
      "-db-config-app-unixsocket", self.mysqldir+"/mysql.sock",
      "-db-config-app-uname", 'vt_dba',   # use vt_dba as some tests depend on 'drop'
      "-db-config-app-keyspace", "test_keyspace",
      "-db-config-app-shard", "0"
    ]
    if self.memcache:
      memcache = self.mysqldir+"/memcache.sock"
      occ_args.extend(["-rowcache-bin", "memcached"])
      occ_args.extend(["-rowcache-socket", memcache])
      occ_args.extend(["-enable-rowcache"])

    if self.sensitive_mode:
      occ_args.extend(['-queryserver-config-sensitive-mode'])

    self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
    self.vtstdout = open("/tmp/vtocc_stdout.log", "a+")
    self.vtocc = subprocess.Popen(occ_args, stdout=self.vtstdout, stderr=self.vtstderr)
    for i in range(30):
      try:
        self.conn = self.connect()
        self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
        self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log', 'r'))

        def flush():
          utils.run(['curl', '-s', '-N', 'http://localhost:9461/debug/flushlogs'], trap_output=True)

        self.log = framework.Tailer(open('/tmp/vtocc.INFO'), flush=flush)
        utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open('/tmp/vtocc_streamlog_9461.log', 'w'))
        self.querylog = framework.Tailer(open('/tmp/vtocc_streamlog_9461.log'), sleep=0.1)
        return
      except dbexceptions.OperationalError:
        if i == 29:
          raise
        time.sleep(1)
Example #2
0
options = None
devnull = open('/dev/null', 'w')
hostname = socket.gethostname()

# binlog_player_protocol_flags defines the flags to use for the binlog players.
# A test can overwrite these flags before calling utils.main().
binlog_player_protocol_flags = ['-binlog_player_protocol', 'gorpc']

class TestError(Exception):
  pass

class Break(Exception):
  pass

environment.setup()

class LoggingStream(object):
  def __init__(self):
    self.line = ""

  def write(self, value):
    if value == "\n":
      # we already printed it
      self.line = ""
      return
    self.line += value
    logging.info("===== " + self.line)
    if value.endswith("\n"):
      self.line = ""
Example #3
0
  def setUp(self):
    utils.zk_setup()
    environment.setup()
    if self.vttop is None:
      raise EnvironmentError("VTTOP not defined")
    if self.vtroot is None:
      raise EnvironmentError("VTROOT not defined")

    framework.execute('go install', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')

    utils.wait_procs([self.tablet.init_mysql()])
    self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])

    self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
    self.clean_sqls = []
    self.init_sqls = []
    clean_mode = False
    with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
      for line in f:
        line = line.rstrip()
        if line == "# clean":
          clean_mode = True
        if line=='' or line.startswith("#"):
          continue
        if clean_mode:
          self.clean_sqls.append(line)
        else:
          self.init_sqls.append(line)
    try:
      for line in self.init_sqls:
        mcu.execute(line, {})
    finally:
      mcu.close()

    utils.run_vtctl('CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
    self.tablet.init_tablet('master', 'test_keyspace', '0')

    customrules = '/tmp/customrules.json'
    self.create_customrules(customrules)
    schema_override = '/tmp/schema_override.json'
    self.create_schema_override(schema_override)
    if self.memcache:
      self.tablet.start_vttablet(memcache=True, customrules=customrules, schema_override=schema_override, sensitive_mode=self.sensitive_mode)
    else:
      self.tablet.start_vttablet(customrules=customrules, schema_override=schema_override, sensitive_mode=self.sensitive_mode)

    # FIXME(szopa): This is necessary here only because of a bug that
    # makes the qs reload its config only after an action.
    utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)

    for i in range(30):
      try:
        self.conn = self.connect()
        self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
        self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'), flush=self.tablet.flush)
        self.log = framework.Tailer(open(os.path.join(environment.vtlogroot, 'vttablet.INFO')), flush=self.tablet.flush)
        querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
        utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open(querylog_file, 'w'))
        time.sleep(1)
        self.querylog = framework.Tailer(open(querylog_file), sleep=0.1)

        return
      except dbexceptions.OperationalError:
        if i == 29:
          raise
        time.sleep(1)