def _insert_values(self, vttablet, id_offset, msg, keyspace_id, num_values): """Inserts values into MySQL along with the required routing comments. Args: vttablet: the Tablet instance to modify. id_offset: offset for the value of `id` column. msg: the value of `msg` column. keyspace_id: the value of `keyspace_id` column. num_values: number of rows to be inserted. """ # For maximum performance, multiple values are inserted in one statement. # However, when the statements are too long, queries will timeout and # vttablet will kill them. Therefore, we chunk it into multiple statements. def chunks(full_list, n): """Yield successive n-sized chunks from full_list.""" for i in xrange(0, len(full_list), n): yield full_list[i:i+n] max_chunk_size = 100*1000 k = utils.uint64_to_hex(keyspace_id) for chunk in chunks(range(1, num_values+1), max_chunk_size): logging.debug('Inserting values for range [%d, %d].', chunk[0], chunk[-1]) values_str = '' for i in chunk: if i != chunk[0]: values_str += ',' values_str += "(%d, '%s', 0x%x)" % (id_offset + i, msg, keyspace_id) vttablet.mquery( 'vt_test_keyspace', [ 'begin', 'insert into worker_test(id, msg, keyspace_id) values%s ' '/* vtgate:: keyspace_id:%s */' % (values_str, k), 'commit'], write=True)
def _insert_multi_value(self, tablet_obj, table, mids, msgs, keyspace_ids): """Generate multi-shard insert statements.""" comma_sep = ',' querystr = ('insert into %s(parent_id, id, msg, custom_ksid_col) values' %(table)) values_str = '' id_str = '/* id:' ksid_str = '' for mid, msg, keyspace_id in zip(mids, msgs, keyspace_ids): ksid_str += utils.uint64_to_hex(keyspace_id)+comma_sep values_str += ('(%d, %d, "%s", 0x%x)' % (fixed_parent_id, mid, msg, keyspace_id) + comma_sep) id_str += '%d' % (mid) + comma_sep values_str = values_str.rstrip(comma_sep) values_str += '/* vtgate:: keyspace_id:%s */ ' %(ksid_str.rstrip(comma_sep)) values_str += id_str.rstrip(comma_sep) + '*/' querystr += values_str tablet_obj.mquery( 'vt_test_keyspace', ['begin', querystr, 'commit'], write=True)
def _insert_multi_value(self, tablet_obj, table, mids, msgs, keyspace_ids): """Generate multi-shard insert statements.""" comma_sep = ',' querystr = ( 'insert into %s(parent_id, id, msg, custom_ksid_col) values' % (table)) values_str = '' id_str = '/* id:' ksid_str = '' for mid, msg, keyspace_id in zip(mids, msgs, keyspace_ids): ksid_str += utils.uint64_to_hex(keyspace_id) + comma_sep values_str += ('(%d, %d, "%s", 0x%x)' % (fixed_parent_id, mid, msg, keyspace_id) + comma_sep) id_str += '%d' % (mid) + comma_sep values_str = values_str.rstrip(comma_sep) values_str += '/* vtgate:: keyspace_id:%s */ ' % ( ksid_str.rstrip(comma_sep)) values_str += id_str.rstrip(comma_sep) + '*/' querystr += values_str tablet_obj.mquery(tablet_obj.dbname, ['begin', querystr, 'commit'], write=True)
def _insert_value(self, tablet, table, id, msg, keyspace_id): k = utils.uint64_to_hex(keyspace_id) tablet.mquery('vt_test_keyspace', [ 'begin', 'insert into %s(id, msg, keyspace_id) ' 'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */' % (table, id, msg, keyspace_id, k, id), 'commit' ], write=True)
def _insert_value(self, tablet_obj, table, mid, msg, keyspace_id): k = utils.uint64_to_hex(keyspace_id) tablet_obj.mquery('vt_test_keyspace', [ 'begin', 'insert into %s(id, msg, custom_ksid_col) ' 'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ ' '/* id:%d */' % (table, mid, msg, keyspace_id, k, mid), 'commit' ], write=True)
def _insert_value(self, tablet, table, id, msg, keyspace_id): k = utils.uint64_to_hex(keyspace_id) tablet.mquery( 'vt_test_keyspace', ['begin', 'insert into %s(id, msg, keyspace_id) ' 'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */' % (table, id, msg, keyspace_id, k, id), 'commit'], write=True)
def _insert_value(self, tablet_obj, table, mid, msg, keyspace_id): k = utils.uint64_to_hex(keyspace_id) tablet_obj.mquery( 'vt_test_keyspace', ['begin', 'insert into %s(id, msg, custom_ksid_col) ' 'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ ' '/* id:%d */' % (table, mid, msg, keyspace_id, k, mid), 'commit'], write=True)
def _insert_value(self, tablet, table, id, msg, keyspace_id): k = utils.uint64_to_hex(keyspace_id) tablet.mquery( "vt_test_keyspace", [ "begin", "insert into %s(id, msg, keyspace_id) " 'values(%d, "%s", 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */' % (table, id, msg, keyspace_id, k, id), "commit", ], write=True, )
def __init__(self, tablet, object_name, user_id, keyspace_id): threading.Thread.__init__(self) self.tablet = tablet self.object_name = object_name self.user_id = user_id self.keyspace_id = keyspace_id self.str_keyspace_id = utils.uint64_to_hex(keyspace_id) self.done = False self.tablet.mquery('vt_test_keyspace', [ 'begin', 'insert into timestamps(name, time_milli, keyspace_id) ' "values('%s', %d, 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */" % (self.object_name, long(time.time() * 1000), self.keyspace_id, self.str_keyspace_id, self.user_id), 'commit' ], write=True, user='******') self.start()
def __init__(self, tablet, object_name, user_id, keyspace_id): threading.Thread.__init__(self) self.tablet = tablet self.object_name = object_name self.user_id = user_id self.keyspace_id = keyspace_id self.str_keyspace_id = utils.uint64_to_hex(keyspace_id) self.done = False self.tablet.mquery( 'vt_test_keyspace', ['begin', 'insert into timestamps(name, time_milli, keyspace_id) ' "values('%s', %d, 0x%x) /* vtgate:: keyspace_id:%s */ /* user_id:%d */" % (self.object_name, long(time.time() * 1000), self.keyspace_id, self.str_keyspace_id, self.user_id), 'commit'], write=True, user='******') self.start()
def __init__(self, tablet_obj, thread_name, thread_id, user_id, keyspace_id): threading.Thread.__init__(self) self.tablet = tablet_obj self.thread_name = thread_name self.thread_id = thread_id self.user_id = user_id self.keyspace_id = keyspace_id self.str_keyspace_id = utils.uint64_to_hex(keyspace_id) self.done = False self.tablet.mquery('vt_test_keyspace', [ 'begin', 'insert into timestamps(id, time_milli, custom_ksid_col) ' 'values(%d, %d, 0x%x) ' '/* vtgate:: keyspace_id:%s */ /* user_id:%d */' % (self.thread_id, long(time.time() * 1000), self.keyspace_id, self.str_keyspace_id, self.user_id), 'commit' ], write=True, user='******') self.start()
def __init__(self, tablet_obj, thread_name, thread_id, user_id, keyspace_id): threading.Thread.__init__(self) self.tablet = tablet_obj self.thread_name = thread_name self.thread_id = thread_id self.user_id = user_id self.keyspace_id = keyspace_id self.str_keyspace_id = utils.uint64_to_hex(keyspace_id) self.done = False self.tablet.mquery( 'vt_test_keyspace', ['begin', 'insert into timestamps(id, time_milli, custom_ksid_col) ' 'values(%d, %d, 0x%x) ' '/* vtgate:: keyspace_id:%s */ /* user_id:%d */' % (self.thread_id, long(time.time() * 1000), self.keyspace_id, self.str_keyspace_id, self.user_id), 'commit'], write=True, user='******') self.start()
def _insert_multi_value(self, tablet_obj, table, midList, msgList, keyspace_idList): comma_sep = ',' querystr = 'insert into %s(parent_id, id, msg, custom_ksid_col) values' % ( table) values_str = '' id_str = '/* id:' ksid_str = '' for mid, msg, keyspace_id in zip(midList, msgList, keyspace_idList): ksid_str += utils.uint64_to_hex(keyspace_id) + comma_sep values_str += '(%d, %d, "%s", 0x%x)' % (fixed_parent_id, mid, msg, keyspace_id) + comma_sep id_str += '%d' % (mid) + comma_sep values_str = values_str.rstrip( comma_sep) + '/* vtgate:: keyspace_id:%s */ ' % ( ksid_str.rstrip(comma_sep)) values_str += id_str.rstrip(comma_sep) + '*/' querystr += values_str tablet_obj.mquery('vt_test_keyspace', ['begin', querystr, 'commit'], write=True)
def __init__(self, tablet_obj, object_name, user_id, keyspace_id): threading.Thread.__init__(self) self.tablet = tablet_obj self.object_name = object_name self.user_id = user_id self.keyspace_id = keyspace_id self.str_keyspace_id = utils.uint64_to_hex(keyspace_id) self.done = False self.tablet.mquery( "vt_test_keyspace", [ "begin", "insert into timestamps(name, time_milli, keyspace_id) " "values('%s', %d, 0x%x) " "/* vtgate:: keyspace_id:%s */ /* user_id:%d */" % (self.object_name, long(time.time() * 1000), self.keyspace_id, self.str_keyspace_id, self.user_id), "commit", ], write=True, user="******", ) self.start()