def create_shard_routing(class_, is_dml, **kargs): routing = shard_routing.ShardRouting(keyspace) routing.shard_name = kargs.get('shard_name') if routing.shard_name is None: dbexceptions.InternalError("For custom sharding, shard_name cannot be None.") if (_is_iterable_container(routing.shard_name) and is_dml): raise dbexceptions.InternalError( "Writes are not allowed on multiple shards.") return routing
def create_shard_routing(class_, *pargs, **kwargs): routing = db_object.ShardRouting(class_.keyspace) routing.shard_name = kwargs.get('shard_name') if routing.shard_name is None: dbexceptions.InternalError( 'For custom sharding, shard_name cannot be None.') if (_is_iterable_container(routing.shard_name) and is_dml): raise dbexceptions.InternalError( 'Writes are not allowed on multiple shards.') return routing
class DBObjectRangeSharded(DBObjectBase): """Base class for range-sharded db classes. This provides default implementation of routing helper methods, cursor creation and common database access operations. This abstracts sharding information and provides helper methods for common database access operations. """ keyspace = None sharding = shard_constants.RANGE_SHARDED table_name = None columns_list = None id_column_name = 'id' sharding_key_column_name = None entity_id_columns = None @classmethod def create_shard_routing(class_, is_dml, **kargs): routing = ShardRouting(class_.keyspace) keyrange = kargs.get("keyrange", None) if keyrange is not None: if is_dml: dbexceptions.InternalError( "Writes require unique sharding_key and not keyrange.") routing.keyrange = keyrange return routing routing.sharding_key = kargs.get('sharding_key', None) if routing.sharding_key is None: try: entity_id_column = kargs['entity_id_column'] entity_id = kargs['entity_id'] # this may involve a lookup of the index from db. # consider caching it at the app layer for performance. entity_id_sharding_key_map = class_.map_entity_id_sharding_key( entity_id_column, entity_id) routing.entity_id_sharding_key_map = entity_id_sharding_key_map routing = entity_id_sharding_key_map.values() except KeyError, e: raise dbexceptions.ProgrammingError( "For sharded table, sharding_key and entity_id cannot both be empty.") if not class_.is_sharding_key_valid(routing.sharding_key): raise dbexceptions.InternalError("Invalid sharding_key %s" % sharding_key) if (_is_iterable_container(routing.sharding_key) and is_dml): raise dbexceptions.InternalError( "Writes are not allowed on multiple sharding_keys.") return routing
def create_shard_routing(class_, is_dml, **kargs): routing = ShardRouting(class_.keyspace) keyrange = kargs.get("keyrange", None) if keyrange is not None: if is_dml: dbexceptions.InternalError( "Writes require unique sharding_key and not keyrange.") routing.keyrange = keyrange return routing routing.sharding_key = kargs.get('sharding_key', None) if routing.sharding_key is None: try: entity_id_column = kargs['entity_id_column'] entity_id = kargs['entity_id'] # this may involve a lookup of the index from db. # consider caching it at the app layer for performance. entity_id_sharding_key_map = class_.map_entity_id_sharding_key( entity_id_column, entity_id) routing.entity_id_sharding_key_map = entity_id_sharding_key_map routing = entity_id_sharding_key_map.values() except KeyError, e: raise dbexceptions.ProgrammingError( "For sharded table, sharding_key and entity_id cannot both be empty.")
def create_shard_routing(class_, *pargs, **kargs): """This creates the ShardRouting object based on the kargs. This prunes the routing kargs so as not to interfere with the actual database method. Args: *pargs: Positional arguments **kargs: Routing key-value params. These are used to determine routing. There are two mutually exclusive mechanisms to indicate routing. 1. entity_id_map {"entity_id_column": entity_id_value} where entity_id_column could be the sharding key or a lookup based entity column of this table. This helps determine the keyspace_ids for the cursor. 2. keyrange - This helps determine the keyrange for the cursor. Returns: ShardRouting object and modified kargs """ lookup_cursor_method = pargs[0] routing = db_object.ShardRouting(class_.keyspace) entity_id_map = None entity_id_map = kargs.get("entity_id_map", None) if entity_id_map is None: kr = None key_range = kargs.get("keyrange", None) if isinstance(key_range, keyrange.KeyRange): kr = key_range else: kr = keyrange.KeyRange(key_range) if kr is not None: routing.keyrange = kr # Both entity_id_map and keyrange have been evaluated. Return. return routing # entity_id_map is not None if len(entity_id_map) != 1: dbexceptions.ProgrammingError("Invalid entity_id_map '%s'" % entity_id_map) entity_id_col = entity_id_map.keys()[0] entity_id = entity_id_map[entity_id_col] #TODO: the current api means that if a table doesn't have the sharding key column name # then it cannot pass in sharding key for routing purposes. Will this cause # extra load on lookup db/cache ? This is cleaner from a design perspective. if entity_id_col == class_.sharding_key_column_name: # Routing using sharding key. routing.sharding_key = entity_id if not class_.is_sharding_key_valid(routing.sharding_key): raise dbexceptions.InternalError("Invalid sharding_key %s" % routing.sharding_key) else: # Routing using lookup based entity. routing.entity_column_name = entity_id_col routing.entity_id_sharding_key_map = class_.lookup_sharding_key_from_entity_id( lookup_cursor_method, entity_id_col, entity_id) return routing
def create_vtgate_cursor(class_, vtgate_conn, tablet_type, is_dml, **cursor_kargs): cursor_method = functools.partial(db_object.create_cursor_from_params, vtgate_conn, tablet_type, False) routing = class_.create_shard_routing(cursor_method, **cursor_kargs) if is_dml: if routing.sharding_key is None or db_object._is_iterable_container( routing.sharding_key): dbexceptions.InternalError( "Writes require unique sharding_key") keyspace_ids = None keyranges = None if routing.sharding_key is not None: keyspace_ids = [] if db_object._is_iterable_container(routing.sharding_key): for sk in routing.sharding_key: kid = class_.sharding_key_to_keyspace_id(sk) keyspace_ids.append(pack_keyspace_id(kid)) else: kid = class_.sharding_key_to_keyspace_id(routing.sharding_key) keyspace_ids = [ pack_keyspace_id(kid), ] elif routing.entity_id_sharding_key_map is not None: keyspace_ids = [] for sharding_key in routing.entity_id_sharding_key_map.values(): keyspace_ids.append( pack_keyspace_id( class_.sharding_key_to_keyspace_id(sharding_key))) elif routing.keyrange: keyranges = [ routing.keyrange, ] cursor = vtgate_cursor.VTGateCursor(vtgate_conn, class_.keyspace, tablet_type, keyspace_ids=keyspace_ids, keyranges=keyranges, writable=is_dml) cursor.routing = routing return cursor