def page_over_shards(Model, cursor, limit, get_results=lambda q: q.all()):
# TODO revisit passing lambda, and cursor format
cursor = int(cursor)
start_shard_id = engine_manager.shard_key_for_id(cursor)
results = []
remaining_limit = limit
next_cursor = None
for shard_id in sorted(engine_manager.engines):
if shard_id < start_shard_id:
continue
if len(results) >= limit:
break
with session_scope_by_shard_id(shard_id) as mailsync_session:
latest_cursor = cursor if shard_id == start_shard_id else None
query = mailsync_session.query(Model)
if latest_cursor:
query = query.filter(Model.id > latest_cursor)
query = query.order_by(asc(Model.id)).limit(remaining_limit)
latest_results = get_results(query)
if latest_results:
results.extend(latest_results)
last = latest_results[-1]
if hasattr(last, "id"):
next_cursor = last.id
elif "id" in last:
next_cursor = last["id"]
else:
raise ValueError(
"Results returned from get_query must" "have an id"
)
# Handle invalid ids
cursor_implied_shard = next_cursor >> 48
if shard_id != 0 and cursor_implied_shard == 0:
next_cursor += shard_id << 48
remaining_limit -= len(latest_results)
return results, str(next_cursor)
def page_over_shards(Model, cursor, limit, get_results=lambda q: q.all()):
# TODO revisit passing lambda, and cursor format
cursor = int(cursor)
start_shard_id = engine_manager.shard_key_for_id(cursor)
results = []
remaining_limit = limit
next_cursor = None
for shard_id in sorted(engine_manager.engines):
if shard_id < start_shard_id:
continue
if len(results) >= limit:
break
with session_scope_by_shard_id(shard_id) as mailsync_session:
latest_cursor = cursor if shard_id == start_shard_id else None
query = mailsync_session.query(Model)
if latest_cursor:
query = query.filter(Model.id > latest_cursor)
query = query.order_by(asc(Model.id)).limit(remaining_limit)
latest_results = get_results(query)
if latest_results:
results.extend(latest_results)
last = latest_results[-1]
if hasattr(last, 'id'):
next_cursor = last.id
elif 'id' in last:
next_cursor = last['id']
else:
raise ValueError('Results returned from get_query must'
'have an id')
# Handle invalid ids
cursor_implied_shard = next_cursor >> 48
if shard_id != 0 and cursor_implied_shard == 0:
next_cursor += shard_id << 48
remaining_limit -= len(latest_results)
return results, str(next_cursor)
def id_chooser(query, ident):
# STOPSHIP(emfree): is ident a tuple here???
# TODO[k]: What if len(list) > 1?
if isinstance(ident, list) and len(ident) == 1:
ident = ident[0]
return [str(engine_manager.shard_key_for_id(ident))]
def shard_chooser(mapper, instance, clause=None):
return str(engine_manager.shard_key_for_id(instance.id))
