def page_over_shards(Model, cursor, limit, get_results=lambda q: q.all()): # TODO revisit passing lambda, and cursor format cursor = int(cursor) start_shard_id = engine_manager.shard_key_for_id(cursor) results = [] remaining_limit = limit next_cursor = None for shard_id in sorted(engine_manager.engines): if shard_id < start_shard_id: continue if len(results) >= limit: break with session_scope_by_shard_id(shard_id) as mailsync_session: latest_cursor = cursor if shard_id == start_shard_id else None query = mailsync_session.query(Model) if latest_cursor: query = query.filter(Model.id > latest_cursor) query = query.order_by(asc(Model.id)).limit(remaining_limit) latest_results = get_results(query) if latest_results: results.extend(latest_results) last = latest_results[-1] if hasattr(last, "id"): next_cursor = last.id elif "id" in last: next_cursor = last["id"] else: raise ValueError( "Results returned from get_query must" "have an id" ) # Handle invalid ids cursor_implied_shard = next_cursor >> 48 if shard_id != 0 and cursor_implied_shard == 0: next_cursor += shard_id << 48 remaining_limit -= len(latest_results) return results, str(next_cursor)
def page_over_shards(Model, cursor, limit, get_results=lambda q: q.all()): # TODO revisit passing lambda, and cursor format cursor = int(cursor) start_shard_id = engine_manager.shard_key_for_id(cursor) results = [] remaining_limit = limit next_cursor = None for shard_id in sorted(engine_manager.engines): if shard_id < start_shard_id: continue if len(results) >= limit: break with session_scope_by_shard_id(shard_id) as mailsync_session: latest_cursor = cursor if shard_id == start_shard_id else None query = mailsync_session.query(Model) if latest_cursor: query = query.filter(Model.id > latest_cursor) query = query.order_by(asc(Model.id)).limit(remaining_limit) latest_results = get_results(query) if latest_results: results.extend(latest_results) last = latest_results[-1] if hasattr(last, 'id'): next_cursor = last.id elif 'id' in last: next_cursor = last['id'] else: raise ValueError('Results returned from get_query must' 'have an id') # Handle invalid ids cursor_implied_shard = next_cursor >> 48 if shard_id != 0 and cursor_implied_shard == 0: next_cursor += shard_id << 48 remaining_limit -= len(latest_results) return results, str(next_cursor)
def id_chooser(query, ident): # STOPSHIP(emfree): is ident a tuple here??? # TODO[k]: What if len(list) > 1? if isinstance(ident, list) and len(ident) == 1: ident = ident[0] return [str(engine_manager.shard_key_for_id(ident))]
def shard_chooser(mapper, instance, clause=None): return str(engine_manager.shard_key_for_id(instance.id))