Exemple #1
0
    def setUp(self):
        if self.vttop is None:
            raise EnvironmentError("VTTOP not defined")
        if self.vtroot is None:
            raise EnvironmentError("VTROOT not defined")
        try:
            os.makedirs(utils.tmp_root)
        except OSError:
            pass

        utils.setup()

        framework.execute('go build',
                          verbose=utils.options.verbose,
                          cwd=self.vttop + '/go/cmd/vtocc')
        framework.execute('go build',
                          verbose=utils.options.verbose,
                          cwd=self.vttop + '/go/cmd/mysqlctl')

        # start mysql
        res = subprocess.call([
            self.vtroot + "/bin/mysqlctl", "-tablet-uid", self.tabletuid,
            "-port",
            str(self.vtoccport), "-mysql-port",
            str(self.mysqlport), "init"
        ])
        if res != 0:
            raise EnvironmentError("Cannot start mysql")
        res = subprocess.call([
            "mysql", "-S", self.mysqldir + "/mysql.sock", "-u", "vt_dba", "-e",
            "create database vt_test_keyspace ; set global read_only = off"
        ])
        if res != 0:
            raise Exception("Cannot create vt_test_keyspace database")
        dbconfig = self.mysqldir + "/dbconf.json"
        with open(dbconfig, 'w') as f:
            conf = {
                'charset': 'utf8',
                'dbname': 'vt_test_keyspace',
                'host': 'localhost',
                'unix_socket': self.mysqldir + "/mysql.sock",
                'uname': 'vt_dba',  # use vt_dba as some tests depend on 'drop'
                'keyspace': 'test_keyspace',
                'shard': '0',
            }
            json.dump(conf, f)

        self.mysql_conn = self.mysql_connect()
        mcu = self.mysql_conn.cursor()
        self.clean_sqls = []
        self.init_sqls = []
        clean_mode = False
        with open(
                os.path.join(self.vttop, "test", "test_data",
                             "test_schema.sql")) as f:
            for line in f:
                line = line.rstrip()
                if line == "# clean":
                    clean_mode = True
                if line == '' or line.startswith("#"):
                    continue
                if clean_mode:
                    self.clean_sqls.append(line)
                else:
                    self.init_sqls.append(line)
        try:
            for line in self.init_sqls:
                mcu.execute(line, {})
        finally:
            mcu.close()

        customrules = '/tmp/customrules.json'
        self.create_customrules(customrules)
        schema_override = '/tmp/schema_override.json'
        self.create_schema_override(schema_override)

        occ_args = [
            self.vtroot + "/bin/vtocc",
            "-port",
            "9461",
            "-dbconfig",
            dbconfig,
            "-customrules",
            customrules,
            "-schema-override",
            schema_override,
        ]
        if utils.options.memcache:
            memcache = self.mysqldir + "/memcache.sock"
            config = self.mysqldir + "/config.json"
            with open(config, 'w') as f:
                json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
            occ_args.extend(["-queryserver-config-file", config])

        self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
        self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
        for i in range(30):
            try:
                self.conn = self.connect()
                self.txlogger = subprocess.Popen(
                    ['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'],
                    stdout=open('/tmp/vtocc_txlog.log', 'w'))
                self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log',
                                                   'r'))

                def flush():
                    utils.run([
                        'curl', '-s', '-N',
                        'http://localhost:9461/debug/flushlogs'
                    ],
                              trap_output=True)

                self.log = framework.Tailer(open('/tmp/vtocc.INFO'),
                                            flush=flush)
                utils.run_bg([
                    'curl', '-s', '-N',
                    'http://localhost:9461/debug/querylog?full=true'
                ],
                             stdout=open('/tmp/vtocc_streamlog_9461.log', 'w'))
                self.querylog = framework.Tailer(
                    open('/tmp/vtocc_streamlog_9461.log'), sleep=0.1)
                return
            except dbexceptions.OperationalError:
                if i == 29:
                    raise
                time.sleep(1)
Exemple #2
0
    def setUp(self):
        utils.zk_setup()
        utils.setup()
        if self.vttop is None:
            raise EnvironmentError("VTTOP not defined")
        if self.vtroot is None:
            raise EnvironmentError("VTROOT not defined")

        framework.execute('go build',
                          verbose=utils.options.verbose,
                          cwd=self.vttop + '/go/cmd/mysqlctl')

        utils.wait_procs([self.tablet.init_mysql()])
        self.tablet.mquery(
            "",
            ["create database vt_test_keyspace", "set global read_only = off"])

        self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
        self.clean_sqls = []
        self.init_sqls = []
        clean_mode = False
        with open(
                os.path.join(self.vttop, "test", "test_data",
                             "test_schema.sql")) as f:
            for line in f:
                line = line.rstrip()
                if line == "# clean":
                    clean_mode = True
                if line == '' or line.startswith("#"):
                    continue
                if clean_mode:
                    self.clean_sqls.append(line)
                else:
                    self.init_sqls.append(line)
        try:
            for line in self.init_sqls:
                mcu.execute(line, {})
        finally:
            mcu.close()

        utils.run_vtctl(
            'CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
        self.tablet.init_tablet('master', 'test_keyspace', '0')

        customrules = '/tmp/customrules.json'
        self.create_customrules(customrules)
        schema_override = '/tmp/schema_override.json'
        self.create_schema_override(schema_override)
        if utils.options.memcache:
            self.tablet.start_vttablet(memcache=True,
                                       customrules=customrules,
                                       schema_override=schema_override)
        else:
            self.tablet.start_vttablet(customrules=customrules,
                                       schema_override=schema_override)

        # FIXME(szopa): This is necessary here only because of a bug that
        # makes the qs reload its config only after an action.
        utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)

        for i in range(30):
            try:
                self.conn = self.connect()
                self.txlogger = subprocess.Popen(
                    ['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'],
                    stdout=open('/tmp/vtocc_txlog.log', 'w'))
                self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'),
                                              flush=self.tablet.flush)
                self.log = framework.Tailer(open(
                    os.path.join(self.tablet.tablet_dir, 'vttablet.INFO')),
                                            flush=self.tablet.flush)
                querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
                utils.run_bg([
                    'curl', '-s', '-N',
                    'http://localhost:9461/debug/querylog?full=true'
                ],
                             stdout=open(querylog_file, 'w'))
                time.sleep(1)
                self.querylog = framework.Tailer(open(querylog_file),
                                                 sleep=0.1)

                return
            except dbexceptions.OperationalError:
                if i == 29:
                    raise
                time.sleep(1)
Exemple #3
0
  def setUp(self):
    if self.vttop is None:
      raise EnvironmentError("VTTOP not defined")
    if self.vtroot is None:
      raise EnvironmentError("VTROOT not defined")
    try:
      os.makedirs(utils.tmp_root)
    except OSError:
      pass

    utils.setup()

    framework.execute('go build', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/vtocc')
    framework.execute('go build', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')

    # start mysql
    res = subprocess.call([
        self.vtroot+"/bin/mysqlctl",
        "-tablet-uid",  self.tabletuid,
        "-port", str(self.vtoccport),
        "-mysql-port", str(self.mysqlport),
        "init"
        ])
    if res != 0:
      raise EnvironmentError("Cannot start mysql")
    res = subprocess.call([
        "mysql",
        "-S",  self.mysqldir+"/mysql.sock",
        "-u", "vt_dba",
        "-e", "create database vt_test_keyspace ; set global read_only = off"])
    if res != 0:
      raise Exception("Cannot create vt_test_keyspace database")
    dbconfig = self.mysqldir+"/dbconf.json"
    with open(dbconfig, 'w') as f:
      conf = {
          'charset': 'utf8',
          'dbname': 'vt_test_keyspace',
          'host': 'localhost',
          'unix_socket': self.mysqldir+"/mysql.sock",
          'uname': 'vt_dba',   # use vt_dba as some tests depend on 'drop'
          'keyspace': 'test_keyspace',
          'shard' : '0',
          }
      json.dump(conf, f)

    self.mysql_conn = self.mysql_connect()
    mcu = self.mysql_conn.cursor()
    self.clean_sqls = []
    self.init_sqls = []
    clean_mode = False
    with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
      for line in f:
        line = line.rstrip()
        if line == "# clean":
          clean_mode = True
        if line=='' or line.startswith("#"):
          continue
        if clean_mode:
          self.clean_sqls.append(line)
        else:
          self.init_sqls.append(line)
    try:
      for line in self.init_sqls:
        mcu.execute(line, {})
    finally:
      mcu.close()

    customrules = '/tmp/customrules.json'
    self.create_customrules(customrules)
    schema_override = '/tmp/schema_override.json'
    self.create_schema_override(schema_override)

    occ_args = [
      self.vtroot+"/bin/vtocc",
      "-port", "9461",
      "-dbconfig", dbconfig,
      "-customrules", customrules,
      "-schema-override", schema_override,
    ]
    if utils.options.memcache:
      memcache = self.mysqldir+"/memcache.sock"
      config = self.mysqldir+"/config.json"
      with open(config, 'w') as f:
        json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
      occ_args.extend(["-queryserver-config-file", config])

    self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
    self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
    for i in range(30):
      try:
        self.conn = self.connect()
        self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
        self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log', 'r'))

        def flush():
          utils.run(['curl', '-s', '-N', 'http://localhost:9461/debug/flushlogs'], trap_output=True)

        self.log = framework.Tailer(open('/tmp/vtocc.INFO'), flush=flush)
        utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open('/tmp/vtocc_streamlog_9461.log', 'w'))
        self.querylog = framework.Tailer(open('/tmp/vtocc_streamlog_9461.log'), sleep=0.1)
        return
      except dbexceptions.OperationalError:
        if i == 29:
          raise
        time.sleep(1)
Exemple #4
0
  def setUp(self):
    utils.zk_setup()
    utils.setup()
    if self.vttop is None:
      raise EnvironmentError("VTTOP not defined")
    if self.vtroot is None:
      raise EnvironmentError("VTROOT not defined")

    framework.execute('go build', verbose=utils.options.verbose, cwd=self.vttop+'/go/cmd/mysqlctl')

    utils.wait_procs([self.tablet.init_mysql()])
    self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])

    self.mysql_conn, mcu = self.tablet.connect('vt_test_keyspace')
    self.clean_sqls = []
    self.init_sqls = []
    clean_mode = False
    with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
      for line in f:
        line = line.rstrip()
        if line == "# clean":
          clean_mode = True
        if line=='' or line.startswith("#"):
          continue
        if clean_mode:
          self.clean_sqls.append(line)
        else:
          self.init_sqls.append(line)
    try:
      for line in self.init_sqls:
        mcu.execute(line, {})
    finally:
      mcu.close()

    utils.run_vtctl('CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace')
    self.tablet.init_tablet('master', 'test_keyspace', '0')

    customrules = '/tmp/customrules.json'
    self.create_customrules(customrules)
    schema_override = '/tmp/schema_override.json'
    self.create_schema_override(schema_override)
    if utils.options.memcache:
      self.tablet.start_vttablet(memcache=True, customrules=customrules, schema_override=schema_override)
    else:
      self.tablet.start_vttablet(customrules=customrules, schema_override=schema_override)

    # FIXME(szopa): This is necessary here only because of a bug that
    # makes the qs reload its config only after an action.
    utils.run_vtctl('Ping ' + self.tablet.zk_tablet_path)

    for i in range(30):
      try:
        self.conn = self.connect()
        self.txlogger = subprocess.Popen(['curl', '-s', '-N', 'http://localhost:9461/debug/txlog'], stdout=open('/tmp/vtocc_txlog.log', 'w'))
        self.txlog = framework.Tailer(open('/tmp/vtocc_txlog.log'), flush=self.tablet.flush)
        self.log = framework.Tailer(open(os.path.join(self.tablet.tablet_dir, 'vttablet.INFO')), flush=self.tablet.flush)
        querylog_file = '/tmp/vtocc_streamlog_%s.log' % self.tablet.port
        utils.run_bg(['curl', '-s', '-N', 'http://localhost:9461/debug/querylog?full=true'], stdout=open(querylog_file, 'w'))
        time.sleep(1)
        self.querylog = framework.Tailer(open(querylog_file), sleep=0.1)

        return
      except dbexceptions.OperationalError:
        if i == 29:
          raise
        time.sleep(1)
Exemple #5
0
    def setUp(self):
        if self.vttop is None:
            raise EnvironmentError("VTTOP not defined")
        if self.vtroot is None:
            raise EnvironmentError("VTROOT not defined")

        framework.execute("go build", verbose=utils.options.verbose, cwd=self.vttop + "/go/cmd/vtocc")
        framework.execute("go build", verbose=utils.options.verbose, cwd=self.vttop + "/go/cmd/mysqlctl")

        # start mysql
        res = subprocess.call(
            [
                self.vtroot + "/bin/mysqlctl",
                "-tablet-uid",
                self.tabletuid,
                "-port",
                str(self.vtoccport),
                "-mysql-port",
                str(self.mysqlport),
                "init",
            ]
        )
        if res != 0:
            raise EnvironmentError("Cannot start mysql")
        res = subprocess.call(
            [
                "mysql",
                "-S",
                self.mysqldir + "/mysql.sock",
                "-u",
                "vt_dba",
                "-e",
                "create database vt_test_keyspace ; set global read_only = off",
            ]
        )
        if res != 0:
            raise Exception("Cannot create vt_test_keyspace database")
        dbconfig = self.mysqldir + "/dbconf.json"
        with open(dbconfig, "w") as f:
            conf = {
                "charset": "utf8",
                "dbname": "vt_test_keyspace",
                "host": "localhost",
                "unix_socket": self.mysqldir + "/mysql.sock",
                "uname": "vt_dba",  # use vt_dba as some tests depend on 'drop'
                "keyspace": "test_keyspace",
                "shard": "0",
            }
            json.dump(conf, f)

        self.mysql_conn = self.mysql_connect()
        mcu = self.mysql_conn.cursor()
        self.clean_sqls = []
        self.init_sqls = []
        clean_mode = False
        with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
            for line in f:
                line = line.rstrip()
                if line == "# clean":
                    clean_mode = True
                if line == "" or line.startswith("#"):
                    continue
                if clean_mode:
                    self.clean_sqls.append(line)
                else:
                    self.init_sqls.append(line)
        try:
            for line in self.init_sqls:
                mcu.execute(line, {})
        finally:
            mcu.close()

        customrules = "/tmp/customrules.json"
        self.create_customrules(customrules)
        schema_override = "/tmp/schema_override.json"
        self.create_schema_override(schema_override)

        occ_args = [
            self.vtroot + "/bin/vtocc",
            "-port",
            "9461",
            "-dbconfig",
            dbconfig,
            "-logfile",
            self.logfile,
            "-querylog",
            self.querylogfile,
            "-customrules",
            customrules,
            "-schema-override",
            schema_override,
        ]
        if utils.options.memcache:
            memcache = self.mysqldir + "/memcache.sock"
            config = self.mysqldir + "/config.json"
            with open(config, "w") as f:
                json.dump({"RowCache": ["memcached", "-s", memcache]}, f)
            occ_args.extend(["-config", config])

        self.vtstderr = open("/tmp/vtocc_stderr.log", "a+")
        self.vtocc = subprocess.Popen(occ_args, stderr=self.vtstderr)
        for i in range(30):
            try:
                self.conn = self.connect()
                self.querylog = framework.Tailer(open(self.querylogfile, "r"))
                self.log = framework.Tailer(open(self.logfile, "r"))
                self.txlogger = subprocess.Popen(
                    ["curl", "-s", "-N", "http://localhost:9461/debug/txlog"], stdout=open("/tmp/vtocc_txlog.log", "w")
                )
                self.txlog = framework.Tailer(open("/tmp/vtocc_txlog.log", "r"))
                return
            except dbexceptions.OperationalError:
                if i == 29:
                    raise
                time.sleep(1)
Exemple #6
0
    def setUp(self):
        utils.zk_setup()
        if self.vttop is None:
            raise EnvironmentError("VTTOP not defined")
        if self.vtroot is None:
            raise EnvironmentError("VTROOT not defined")

        framework.execute("go build", verbose=utils.options.verbose, cwd=self.vttop + "/go/cmd/mysqlctl")

        utils.wait_procs([self.tablet.init_mysql()])
        self.tablet.mquery("", ["create database vt_test_keyspace", "set global read_only = off"])

        self.mysql_conn, mcu = self.tablet.connect("vt_test_keyspace")
        self.clean_sqls = []
        self.init_sqls = []
        clean_mode = False
        with open(os.path.join(self.vttop, "test", "test_data", "test_schema.sql")) as f:
            for line in f:
                line = line.rstrip()
                if line == "# clean":
                    clean_mode = True
                if line == "" or line.startswith("#"):
                    continue
                if clean_mode:
                    self.clean_sqls.append(line)
                else:
                    self.init_sqls.append(line)
        try:
            for line in self.init_sqls:
                mcu.execute(line, {})
        finally:
            mcu.close()

        utils.run_vtctl("CreateKeyspace -force /zk/global/vt/keyspaces/test_keyspace")
        self.tablet.init_tablet("master", "test_keyspace", "0")

        customrules = "/tmp/customrules.json"
        self.create_customrules(customrules)
        schema_override = "/tmp/schema_override.json"
        self.create_schema_override(schema_override)
        if utils.options.memcache:
            self.tablet.start_vttablet(memcache=True, customrules=customrules, schema_override=schema_override)
        else:
            self.tablet.start_vttablet(customrules=customrules, schema_override=schema_override)

        # FIXME(szopa): This is necessary here only because of a bug that
        # makes the qs reload its config only after an action.
        utils.run_vtctl("Ping " + self.tablet.zk_tablet_path)

        for i in range(30):
            try:
                self.conn = self.connect()
                self.querylog = framework.Tailer(open(self.tablet.querylog_file, "r"))
                self.log = framework.Tailer(open(self.tablet.logfile, "r"))
                self.txlogger = subprocess.Popen(
                    ["curl", "-s", "-N", "http://localhost:9461/debug/txlog"], stdout=open("/tmp/vtocc_txlog.log", "w")
                )
                self.txlog = framework.Tailer(open("/tmp/vtocc_txlog.log", "r"))
                return
            except dbexceptions.OperationalError:
                if i == 29:
                    raise
                time.sleep(1)