def _perform_in_chunks(actions, operation, chunk_size=None):
if chunk_size is None:
from nefertari_es import Settings
chunk_size = Settings.asint('chunk_size', 500)
start = end = 0
count = len(actions)
while count:
if count < chunk_size:
chunk_size = count
end += chunk_size
operation(actions=actions[start:end])
start += chunk_size
count -= chunk_size
def _bulk(actions, client, op_type='index', request=None):
from nefertari_es import Settings
for action in actions:
action['_op_type'] = op_type
kwargs = {
'client': client,
'actions': actions,
}
if request is None:
query_params = {}
else:
query_params = request.params.mixed()
query_params = dictset(query_params)
refresh_enabled = Settings.asbool('enable_refresh_query', False)
if '_refresh_index' in query_params and refresh_enabled:
kwargs['refresh'] = query_params.asbool('_refresh_index')
executed_num, errors = helpers.bulk(**kwargs)
if errors:
raise Exception('Errors happened when executing Elasticsearch '
'actions: {}'.format('; '.join(errors)))
return executed_num