コード例 #1
0
ファイル: worker.py プロジェクト: DalianDragon/vitess
  def _insert_values(self, vttablet, id_offset, msg, keyspace_id, num_values):
    """Inserts values into MySQL along with the required routing comments.

    Args:
      vttablet: the Tablet instance to modify.
      id_offset: offset for the value of `id` column.
      msg: the value of `msg` column.
      keyspace_id: the value of `keyspace_id` column.
      num_values: number of rows to be inserted.
    """

    # For maximum performance, multiple values are inserted in one statement.
    # However, when the statements are too long, queries will timeout and
    # vttablet will kill them. Therefore, we chunk it into multiple statements.
    def chunks(full_list, n):
      """Yield successive n-sized chunks from full_list."""
      for i in xrange(0, len(full_list), n):
        yield full_list[i:i+n]

    max_chunk_size = 100*1000
    k = utils.uint64_to_hex(keyspace_id)
    for chunk in chunks(range(1, num_values+1), max_chunk_size):
      logging.debug('Inserting values for range [%d, %d].', chunk[0], chunk[-1])
      values_str = ''
      for i in chunk:
        if i != chunk[0]:
          values_str += ','
        values_str += "(%d, '%s', 0x%x)" % (id_offset + i, msg, keyspace_id)
      vttablet.mquery(
          'vt_test_keyspace', [
              'begin',
              'insert into worker_test(id, msg, keyspace_id) values%s '
              '/* vtgate:: keyspace_id:%s */' % (values_str, k),
              'commit'],
          write=True)
コード例 #2
0
  def _insert_values(self, vttablet, id_offset, msg, keyspace_id, num_values):
    """Inserts values into MySQL along with the required routing comments.

    Args:
      vttablet: the Tablet instance to modify.
      id_offset: offset for the value of `id` column.
      msg: the value of `msg` column.
      keyspace_id: the value of `keyspace_id` column.
      num_values: number of rows to be inserted.
    """

    # For maximum performance, multiple values are inserted in one statement.
    # However, when the statements are too long, queries will timeout and
    # vttablet will kill them. Therefore, we chunk it into multiple statements.
    def chunks(full_list, n):
      """Yield successive n-sized chunks from full_list."""
      for i in xrange(0, len(full_list), n):
        yield full_list[i:i+n]

    max_chunk_size = 100*1000
    k = utils.uint64_to_hex(keyspace_id)
    for chunk in chunks(range(1, num_values+1), max_chunk_size):
      logging.debug('Inserting values for range [%d, %d].', chunk[0], chunk[-1])
      values_str = ''
      for i in chunk:
        if i != chunk[0]:
          values_str += ','
        values_str += "(%d, '%s', 0x%x)" % (id_offset + i, msg, keyspace_id)
      vttablet.mquery(
          'vt_test_keyspace', [
              'begin',
              'insert into worker_test(id, msg, keyspace_id) values%s '
              '/* vtgate:: keyspace_id:%s */' % (values_str, k),
              'commit'],
          write=True)
コード例 #3
0
ファイル: base_sharding.py プロジェクト: alainjobart/vitess
  def _insert_multi_value(self, tablet_obj, table, mids, msgs, keyspace_ids):
    """Generate multi-shard insert statements."""
    comma_sep = ','
    querystr = ('insert into %s(parent_id, id, msg, custom_ksid_col) values'
                %(table))
    values_str = ''
    id_str = '/* id:'
    ksid_str = ''

    for mid, msg, keyspace_id in zip(mids, msgs, keyspace_ids):
      ksid_str += utils.uint64_to_hex(keyspace_id)+comma_sep
      values_str += ('(%d, %d, "%s", 0x%x)' %
                     (fixed_parent_id, mid, msg, keyspace_id) + comma_sep)
      id_str += '%d' % (mid) + comma_sep

    values_str = values_str.rstrip(comma_sep)
    values_str += '/* vtgate:: keyspace_id:%s */ ' %(ksid_str.rstrip(comma_sep))
    values_str += id_str.rstrip(comma_sep) + '*/'

    querystr += values_str
    tablet_obj.mquery(
        'vt_test_keyspace',
        ['begin',
         querystr,
         'commit'],
        write=True)
コード例 #4
0
    def _insert_multi_value(self, tablet_obj, table, mids, msgs, keyspace_ids):
        """Generate multi-shard insert statements."""
        comma_sep = ','
        querystr = (
            'insert into %s(parent_id, id, msg, custom_ksid_col) values' %
            (table))
        values_str = ''
        id_str = '/* id:'
        ksid_str = ''

        for mid, msg, keyspace_id in zip(mids, msgs, keyspace_ids):
            ksid_str += utils.uint64_to_hex(keyspace_id) + comma_sep
            values_str += ('(%d, %d, "%s", 0x%x)' %
                           (fixed_parent_id, mid, msg, keyspace_id) +
                           comma_sep)
            id_str += '%d' % (mid) + comma_sep

        values_str = values_str.rstrip(comma_sep)
        values_str += '/* vtgate:: keyspace_id:%s */ ' % (
            ksid_str.rstrip(comma_sep))
        values_str += id_str.rstrip(comma_sep) + '*/'

        querystr += values_str
        tablet_obj.mquery(tablet_obj.dbname, ['begin', querystr, 'commit'],
                          write=True)
コード例 #5
0
 def _insert_value(self, tablet, table, id, msg, keyspace_id):
     k = utils.uint64_to_hex(keyspace_id)
     tablet.mquery('vt_test_keyspace', [
         'begin',
         'insert into %s(id, msg, keyspace_id) '
         'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */'
         % (table, id, msg, keyspace_id, k, id), 'commit'
     ],
                   write=True)
コード例 #6
0
 def _insert_value(self, tablet_obj, table, mid, msg, keyspace_id):
     k = utils.uint64_to_hex(keyspace_id)
     tablet_obj.mquery('vt_test_keyspace', [
         'begin',
         'insert into %s(id, msg, custom_ksid_col) '
         'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ '
         '/* id:%d */' % (table, mid, msg, keyspace_id, k, mid), 'commit'
     ],
                       write=True)
コード例 #7
0
ファイル: initial_sharding.py プロジェクト: richarwu/vitess
 def _insert_value(self, tablet, table, id, msg, keyspace_id):
   k = utils.uint64_to_hex(keyspace_id)
   tablet.mquery(
       'vt_test_keyspace',
       ['begin',
        'insert into %s(id, msg, keyspace_id) '
        'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */' %
        (table, id, msg, keyspace_id, k, id),
        'commit'],
       write=True)
コード例 #8
0
ファイル: base_sharding.py プロジェクト: CowLeo/vitess
 def _insert_value(self, tablet_obj, table, mid, msg, keyspace_id):
   k = utils.uint64_to_hex(keyspace_id)
   tablet_obj.mquery(
       'vt_test_keyspace',
       ['begin',
        'insert into %s(id, msg, custom_ksid_col) '
        'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ '
        '/* id:%d */' %
        (table, mid, msg, keyspace_id, k, mid),
        'commit'],
       write=True)
コード例 #9
0
ファイル: resharding.py プロジェクト: hadoop835/vitess
 def _insert_value(self, tablet, table, id, msg, keyspace_id):
     k = utils.uint64_to_hex(keyspace_id)
     tablet.mquery(
         "vt_test_keyspace",
         [
             "begin",
             "insert into %s(id, msg, keyspace_id) "
             'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */'
             % (table, id, msg, keyspace_id, k, id),
             "commit",
         ],
         write=True,
     )
コード例 #10
0
    def __init__(self, tablet, object_name, user_id, keyspace_id):
        threading.Thread.__init__(self)
        self.tablet = tablet
        self.object_name = object_name
        self.user_id = user_id
        self.keyspace_id = keyspace_id
        self.str_keyspace_id = utils.uint64_to_hex(keyspace_id)
        self.done = False

        self.tablet.mquery('vt_test_keyspace', [
            'begin',
            'insert into timestamps(name, time_milli, keyspace_id) '
            "values('%s', %d, 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */"
            % (self.object_name, long(time.time() * 1000), self.keyspace_id,
               self.str_keyspace_id, self.user_id), 'commit'
        ],
                           write=True,
                           user='******')
        self.start()
コード例 #11
0
ファイル: resharding.py プロジェクト: richarwu/vitess
  def __init__(self, tablet, object_name, user_id, keyspace_id):
    threading.Thread.__init__(self)
    self.tablet = tablet
    self.object_name = object_name
    self.user_id = user_id
    self.keyspace_id = keyspace_id
    self.str_keyspace_id = utils.uint64_to_hex(keyspace_id)
    self.done = False

    self.tablet.mquery(
        'vt_test_keyspace',
        ['begin',
         'insert into timestamps(name, time_milli, keyspace_id) '
         "values('%s', %d, 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */" %
         (self.object_name, long(time.time() * 1000), self.keyspace_id,
          self.str_keyspace_id, self.user_id),
         'commit'],
        write=True, user='******')
    self.start()
コード例 #12
0
    def __init__(self, tablet_obj, thread_name, thread_id, user_id,
                 keyspace_id):
        threading.Thread.__init__(self)
        self.tablet = tablet_obj
        self.thread_name = thread_name
        self.thread_id = thread_id
        self.user_id = user_id
        self.keyspace_id = keyspace_id
        self.str_keyspace_id = utils.uint64_to_hex(keyspace_id)
        self.done = False

        self.tablet.mquery('vt_test_keyspace', [
            'begin',
            'insert into timestamps(id, time_milli, custom_ksid_col) '
            'values(%d, %d, 0x%x) '
            '/* vtgate:: keyspace_id:%s */ /* user_id:%d */' %
            (self.thread_id, long(time.time() * 1000), self.keyspace_id,
             self.str_keyspace_id, self.user_id), 'commit'
        ],
                           write=True,
                           user='******')
        self.start()
コード例 #13
0
ファイル: resharding.py プロジェクト: CowLeo/vitess
  def __init__(self, tablet_obj, thread_name, thread_id, user_id,
               keyspace_id):
    threading.Thread.__init__(self)
    self.tablet = tablet_obj
    self.thread_name = thread_name
    self.thread_id = thread_id
    self.user_id = user_id
    self.keyspace_id = keyspace_id
    self.str_keyspace_id = utils.uint64_to_hex(keyspace_id)
    self.done = False

    self.tablet.mquery(
        'vt_test_keyspace',
        ['begin',
         'insert into timestamps(id, time_milli, custom_ksid_col) '
         'values(%d, %d, 0x%x) '
         '/* vtgate:: keyspace_id:%s */ /* user_id:%d */' %
         (self.thread_id, long(time.time() * 1000), self.keyspace_id,
          self.str_keyspace_id, self.user_id),
         'commit'],
        write=True, user='******')
    self.start()
コード例 #14
0
    def _insert_multi_value(self, tablet_obj, table, midList, msgList,
                            keyspace_idList):
        comma_sep = ','
        querystr = 'insert into %s(parent_id, id, msg, custom_ksid_col) values' % (
            table)
        values_str = ''
        id_str = '/* id:'
        ksid_str = ''

        for mid, msg, keyspace_id in zip(midList, msgList, keyspace_idList):
            ksid_str += utils.uint64_to_hex(keyspace_id) + comma_sep
            values_str += '(%d, %d, "%s", 0x%x)' % (fixed_parent_id, mid, msg,
                                                    keyspace_id) + comma_sep
            id_str += '%d' % (mid) + comma_sep

        values_str = values_str.rstrip(
            comma_sep) + '/* vtgate:: keyspace_id:%s */ ' % (
                ksid_str.rstrip(comma_sep))
        values_str += id_str.rstrip(comma_sep) + '*/'

        querystr += values_str
        tablet_obj.mquery('vt_test_keyspace', ['begin', querystr, 'commit'],
                          write=True)
コード例 #15
0
ファイル: resharding.py プロジェクト: hadmagic/vitess
    def __init__(self, tablet_obj, object_name, user_id, keyspace_id):
        threading.Thread.__init__(self)
        self.tablet = tablet_obj
        self.object_name = object_name
        self.user_id = user_id
        self.keyspace_id = keyspace_id
        self.str_keyspace_id = utils.uint64_to_hex(keyspace_id)
        self.done = False

        self.tablet.mquery(
            "vt_test_keyspace",
            [
                "begin",
                "insert into timestamps(name, time_milli, keyspace_id) "
                "values('%s', %d, 0x%x) "
                "/* vtgate:: keyspace_id:%s */ /* user_id:%d */"
                % (self.object_name, long(time.time() * 1000), self.keyspace_id, self.str_keyspace_id, self.user_id),
                "commit",
            ],
            write=True,
            user="******",
        )
        self.start()