def run(self):
from nefertari.elasticsearch import ES
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urlparse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
es.index(documents, chunk_size=chunk_size)
else:
es.index_missing_documents(documents, chunk_size=chunk_size)
return 0
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()])
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
local_params = dict()
local_params.update(params)
if '_limit' not in local_params:
limit = model.get_collection().count()
local_params['_limit'] = limit
chunk_size = int(self.options.chunk or local_params['_limit'])
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**local_params)
documents = to_indexable_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(model_name))
es.index(documents)
else:
self.log.info(
'Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
return 0
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(
model_name))
es.index(documents)
else:
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
return 0
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
def index_models(self, model_names):
self.log.info('Indexing models documents')
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
