def sql_migrate(index, sql, max_id, increment, es_args=None, **kwargs):
""" Run provided SQL and send output to elastic.
:param str index: Elastic index to update (formatted into `sql`)
:param str sql: SQL to format and run. See __init__.py in this module
:param int max_id: Last known object id. Indicates when to stop paging
:param int increment: Page size
:param dict es_args: Dict or None, to pass to `helpers.bulk`
:kwargs: Additional format arguments for `sql` arg
:return int: Number of migrated objects
"""
if es_args is None:
es_args = {}
total_pages = int(ceil(max_id / float(increment)))
total_objs = 0
page_start = 0
page_end = 0
page = 0
while page_end <= (max_id + increment):
page += 1
page_end += increment
if page <= total_pages:
logger.info('Updating page {} / {}'.format(page_end / increment,
total_pages))
else:
# An extra page is included to cover the edge case where:
# max_id == (total_pages * increment) - 1
# and two additional objects are created during runtime.
logger.info('Cleaning up...')
with connection.cursor() as cursor:
cursor.execute(
sql.format(index=index,
page_start=page_start,
page_end=page_end,
**kwargs))
ser_objs = cursor.fetchone()[0]
if ser_objs:
total_objs += len(ser_objs)
helpers.bulk(client(), ser_objs, **es_args)
page_start = page_end
return total_objs
def sql_migrate(index, sql, max_id, increment, es_args=None, **kwargs):
""" Run provided SQL and send output to elastic.
:param str index: Elastic index to update (formatted into `sql`)
:param str sql: SQL to format and run. See __init__.py in this module
:param int max_id: Last known object id. Indicates when to stop paging
:param int increment: Page size
:param dict es_args: Dict or None, to pass to `helpers.bulk`
:kwargs: Additional format arguments for `sql` arg
:return int: Number of migrated objects
"""
if es_args is None:
es_args = {}
total_pages = int(ceil(max_id / float(increment)))
total_objs = 0
page_start = 0
page_end = 0
page = 0
while page_end <= (max_id + increment):
page += 1
page_end += increment
if page <= total_pages:
logger.info('Updating page {} / {}'.format(page_end / increment, total_pages))
else:
# An extra page is included to cover the edge case where:
# max_id == (total_pages * increment) - 1
# and two additional objects are created during runtime.
logger.info('Cleaning up...')
with connection.cursor() as cursor:
cursor.execute(sql.format(
index=index,
page_start=page_start,
page_end=page_end,
**kwargs))
ser_objs = cursor.fetchone()[0]
if ser_objs:
total_objs += len(ser_objs)
helpers.bulk(client(), ser_objs, **es_args)
page_start = page_end
return total_objs