def test_batch_read(self): # TODO(sougou): fix return # 1. Create select queries using DB classes. query_list = [] bv_list = [] user_id_list = [self.user_id_list[0], self.user_id_list[1]] where_column_value_pairs = (('id', user_id_list), ) entity_id_map = dict(where_column_value_pairs) q, bv = db_class_sharded.VtUser.create_select_query( where_column_value_pairs) query_list.append(q) bv_list.append(bv) where_column_value_pairs = (('user_id', user_id_list), ) q, bv = db_class_sharded.VtUserEmail.create_select_query( where_column_value_pairs) query_list.append(q) bv_list.append(bv) with database_context.ReadFromMaster(self.dc) as context: # 2. Cursor Creation using one of the DB classes. cursor = context.get_cursor(entity_id_map=entity_id_map)( db_class_sharded.VtUser) # 3. Batch execution of reads. results = db_object.execute_batch_read(cursor, query_list, bv_list) self.assertEqual(len(results), len(query_list)) res_ids = [row.id for row in results[0]] res_user_ids = [row.user_id for row in results[1]] self.assertEqual(res_ids, user_id_list) self.assertEqual(res_user_ids, user_id_list)
def test_entity_id_read(self): with database_context.ReadFromMaster(self.dc) as context: entity_id_map = {'username': '******'} rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), [('id', self.user_id_list[0]),]) self.assertEqual(len(rows), 1, "wrong number of rows fetched")
def test_in_clause_read(self): with database_context.ReadFromMaster(self.dc) as context: user_id_list = [self.user_id_list[0], self.user_id_list[1]] where_column_value_pairs = (('id', user_id_list),) entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUser.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 2, "wrong number of rows fetched") self.assertEqual(user_id_list, [row.id for row in rows], "wrong rows fetched") username_list = [row.username for row in rows] where_column_value_pairs = (('username', username_list),) entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUser.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 2, "wrong number of rows fetched") self.assertEqual(username_list, [row.username for row in rows], "wrong rows fetched") where_column_value_pairs = (('user_id', user_id_list),) entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUserEmail.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 2, "wrong number of rows fetched") self.assertEqual(user_id_list, [row.user_id for row in rows], "wrong rows fetched")
def test_entity_id_read(self): user_id = self.user_id_list[0] with database_context.ReadFromMaster(self.dc) as context: entity_id_map = {'username': '******'} rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), [ ('id', user_id), ]) self.assertEqual(len(rows), 1, "wrong number of rows fetched") where_column_value_pairs = [ ('id', self.user_song_map[user_id][0]), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtSong.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 1, "wrong number of rows fetched") where_column_value_pairs = [ ('song_id', self.user_song_map[user_id][0]), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtSongDetail.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 1, "wrong number of rows fetched")
def test_sharding_key_read(self): user_id = self.user_id_list[0] with database_context.ReadFromMaster(self.dc) as context: where_column_value_pairs = [ ('id', user_id), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 1, 'wrong number of rows fetched') where_column_value_pairs = [ ('user_id', user_id), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUserEmail.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 1, 'wrong number of rows fetched') where_column_value_pairs = [ ('user_id', user_id), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtSong.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), len(self.user_song_map[user_id]), 'wrong number of rows fetched')
def test_scatter_read(self): where_column_value_pairs = [] with database_context.ReadFromMaster(self.dc) as context: rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(keyrange=keyrange_constants.NON_PARTIAL_KEYRANGE), where_column_value_pairs) self.assertEqual(len(rows), len(self.user_id_list), "wrong number of rows fetched, expecting %d got %d" % (len(self.user_id_list), len(rows)))
def delete_columns(self): user_id = self.user_id_list[-1] with database_context.WriteTransaction(self.dc) as context: where_column_value_pairs = [ ('id', user_id), ] entity_id_map = {'id': user_id} db_class_sharded.VtUser.delete_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) where_column_value_pairs = [ ('user_id', user_id), ] entity_id_map = {'user_id': user_id} db_class_sharded.VtUserEmail.delete_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) with database_context.ReadFromMaster(self.dc) as context: rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 0, "wrong number of rows fetched") rows = db_class_sharded.VtUserEmail.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 0, "wrong number of rows fetched") self.user_id_list = self.user_id_list[:-1] self.user_id_list.sort()
def test_read(self): with database_context.ReadFromMaster(self.dc) as context: rows = db_class_unsharded.VtUnsharded.select_by_id( context.get_cursor(), 2) self.assertEqual(len(rows), 1, "wrong number of rows fetched") for row in rows: logging.info("ROW: %s" % row) self.assertEqual(rows[0].id, 2, "wrong row fetched")
def test_count(self): with database_context.ReadFromMaster(self.dc) as context: count = db_class_unsharded.VtUnsharded.get_count( context.get_cursor(), msg="test message") expected = len(self.all_ids) self.assertEqual( count, expected, "wrong count fetched; expected %d got %d" % (expected, count))
def test_count(self): with database_context.ReadFromMaster(self.dc) as context: count = db_class_sharded.VtUser.get_count(context.get_cursor( keyrange=keyrange_constants.NON_PARTIAL_KEYRANGE), msg="test message") expected = len(self.user_id_list) self.assertEqual( count, expected, "wrong count fetched; expected %d got %d" % (expected, count))
def test_delete_and_read(self): where_column_value_pairs = [('id', 2)] with database_context.WriteTransaction(self.dc) as context: db_class_unsharded.VtUnsharded.delete_by_columns(context.get_cursor(), where_column_value_pairs) with database_context.ReadFromMaster(self.dc) as context: rows = db_class_unsharded.VtUnsharded.select_by_id(context.get_cursor(), 2) self.assertEqual(len(rows), 0, "wrong number of rows fetched")
def test_min_id(self): with database_context.ReadFromMaster(self.dc) as context: min_id = db_class_unsharded.VtUnsharded.get_min( context.get_cursor()) expected = min(self.all_ids) self.assertEqual( min_id, expected, "wrong min value fetched; expected %d got %d" % (expected, min_id))
def test_max_id(self): with database_context.ReadFromMaster(self.dc) as context: max_id = db_class_unsharded.VtUnsharded.get_max( context.get_cursor()) self.all_ids.sort() expected = max(self.all_ids) self.assertEqual( max_id, expected, 'wrong max value fetched; expected %d got %d' % (expected, max_id))
def test_max_id(self): with database_context.ReadFromMaster(self.dc) as context: max_id = db_class_sharded.VtUser.get_max( context.get_cursor( keyrange=keyrange_constants.NON_PARTIAL_KEYRANGE)) expected = max(self.user_id_list) self.assertEqual( max_id, expected, "wrong max value fetched; expected %d got %d" % (expected, max_id))
def test_streaming_read(self): where_column_value_pairs = [] with database_context.ReadFromMaster(self.dc) as context: rows = db_class_sharded.VtUser.select_by_columns_streaming( context.get_cursor(keyrange=keyrange_constants.NON_PARTIAL_KEYRANGE), where_column_value_pairs) got_user_id_list = [] for r in rows: got_user_id_list.append(r.id) self.assertEqual(len(got_user_id_list), len(self.user_id_list), "wrong number of rows fetched")
def test_keyrange_read(self): where_column_value_pairs = [] with database_context.ReadFromMaster(self.dc) as context: rows1 = db_class_sharded.VtUser.select_by_columns( context.get_cursor(keyrange='-80'), where_column_value_pairs) rows2 = db_class_sharded.VtUser.select_by_columns( context.get_cursor(keyrange='80-'), where_column_value_pairs) fetched_rows = len(rows1) + len(rows2) expected = len(self.user_id_list) self.assertEqual(fetched_rows, expected, "wrong number of rows fetched expected:%d got:%d" % (expected, fetched_rows))
def test_read(self): id_val = self.all_ids[0] with database_context.ReadFromMaster(self.dc) as context: rows = db_class_unsharded.VtUnsharded.select_by_id( context.get_cursor(), id_val) expected = 1 self.assertEqual( len(rows), expected, "wrong number of rows fetched %d, expected %d" % (len(rows), expected)) self.assertEqual(rows[0].id, id_val, "wrong row fetched")
def test_sharding_key_read(self): with database_context.ReadFromMaster(self.dc) as context: where_column_value_pairs = [('id', self.user_id_list[0]),] rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(entity_id_map={'id':self.user_id_list[0]}), where_column_value_pairs) self.assertEqual(len(rows), 1, "wrong number of rows fetched") where_column_value_pairs = [('user_id', self.user_id_list[0]),] rows = db_class_sharded.VtUserEmail.select_by_columns( context.get_cursor(entity_id_map={'user_id':self.user_id_list[0]}), where_column_value_pairs) self.assertEqual(len(rows), 1, "wrong number of rows fetched")
def update_columns(self): with database_context.WriteTransaction(self.dc) as context: user_id = self.user_id_list[1] where_column_value_pairs = [ ('id', user_id), ] entity_id_map = {'id': user_id} new_username = '******' % user_id update_cols = [ ('username', new_username), ] db_class_sharded.VtUser.update_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs, update_column_value_pairs=update_cols) # verify the updated value. where_column_value_pairs = [ ('id', user_id), ] rows = db_class_sharded.VtUser.select_by_columns( context.get_cursor(entity_id_map={'id': user_id}), where_column_value_pairs) self.assertEqual(len(rows), 1, "wrong number of rows fetched") self.assertEqual(new_username, rows[0].username) where_column_value_pairs = [ ('user_id', user_id), ] entity_id_map = {'user_id': user_id} new_email = '*****@*****.**' % user_id m = hashlib.md5() m.update(new_email) email_hash = m.digest() update_cols = [('email', new_email), ('email_hash', email_hash)] db_class_sharded.VtUserEmail.update_columns( context.get_cursor(entity_id_map={'user_id': user_id}), where_column_value_pairs, update_column_value_pairs=update_cols) # verify the updated value. with database_context.ReadFromMaster(self.dc) as context: where_column_value_pairs = [ ('user_id', user_id), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUserEmail.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 1, "wrong number of rows fetched") self.assertEqual(new_email, rows[0].email) self.user_id_list.sort()
def test_update_and_read(self): id_val = self.all_ids[0] where_column_value_pairs = [('id', id_val)] with database_context.WriteTransaction(self.dc) as context: db_class_unsharded.VtUnsharded.update_columns( context.get_cursor(), where_column_value_pairs, msg="test update") with database_context.ReadFromMaster(self.dc) as context: rows = db_class_unsharded.VtUnsharded.select_by_id( context.get_cursor(), id_val) self.assertEqual(len(rows), 1, "wrong number of rows fetched") self.assertEqual(rows[0].msg, "test update", "wrong row fetched")
def test_min_id(self): with database_context.ReadFromMaster(self.dc) as context: min_id = db_class_sharded.VtUser.get_min( context.get_cursor( keyrange=keyrange_constants.NON_PARTIAL_KEYRANGE)) self.user_id_list.sort() expected = min(self.user_id_list) rows1 = db_class_sharded.VtUser.select_by_columns( context.get_cursor( keyrange=keyrange_constants.NON_PARTIAL_KEYRANGE), []) id_list = [row.id for row in rows1] self.assertEqual( min_id, expected, "wrong min value fetched; expected %d got %d" % (expected, min_id))
def test_batch_read(self): query_list = [] bv_list = [] user_id_list = [self.user_id_list[0], self.user_id_list[1]] where_column_value_pairs = (('id', user_id_list), ) entity_id_map = dict(where_column_value_pairs) q, bv = db_class_sharded.VtUser.create_select_query( where_column_value_pairs) query_list.append(q) bv_list.append(bv) where_column_value_pairs = (('user_id', user_id_list), ) q, bv = db_class_sharded.VtUserEmail.create_select_query( where_column_value_pairs) query_list.append(q) bv_list.append(bv) with database_context.ReadFromMaster(self.dc) as context: cursor = context.get_cursor(entity_id_map=entity_id_map)( db_class_sharded.VtUser) results = db_object.execute_batch_read(cursor, query_list, bv_list) self.assertEqual(len(results), len(query_list)) res_ids = [row.id for row in results[0]] res_user_ids = [row.user_id for row in results[1]] self.assertEqual(res_ids, user_id_list) self.assertEqual(res_user_ids, user_id_list)
def test_in_clause_read(self): with database_context.ReadFromMaster(self.dc) as context: user_id_list = [self.user_id_list[0], self.user_id_list[1]] where_column_value_pairs = (('id', user_id_list), ) entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUser.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 2, "wrong number of rows fetched") got = [row.id for row in rows] got.sort() self.assertEqual( user_id_list, got, "wrong rows fetched; expected %s got %s" % (user_id_list, got)) username_list = [row.username for row in rows] username_list.sort() where_column_value_pairs = (('username', username_list), ) entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUser.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 2, "wrong number of rows fetched") got = [row.username for row in rows] got.sort() self.assertEqual( username_list, got, "wrong rows fetched; expected %s got %s" % (username_list, got)) where_column_value_pairs = (('user_id', user_id_list), ) entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtUserEmail.select_by_ids( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) self.assertEqual(len(rows), 2, "wrong number of rows fetched") got = [row.user_id for row in rows] got.sort() self.assertEqual( user_id_list, got, "wrong rows fetched; expected %s got %s" % (user_id_list, got)) song_id_list = [] for user_id in user_id_list: song_id_list.extend(self.user_song_map[user_id]) song_id_list.sort() where_column_value_pairs = [ ('id', song_id_list), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtSong.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) got = [row.id for row in rows] got.sort() self.assertEqual( song_id_list, got, "wrong rows fetched %s got %s" % (song_id_list, got)) where_column_value_pairs = [ ('song_id', song_id_list), ] entity_id_map = dict(where_column_value_pairs) rows = db_class_sharded.VtSongDetail.select_by_columns( context.get_cursor(entity_id_map=entity_id_map), where_column_value_pairs) got = [row.song_id for row in rows] got.sort() self.assertEqual( song_id_list, got, "wrong rows fetched %s got %s" % (song_id_list, got))