def run(self, quiet=False):
from nefertari.elasticsearch import ES
ES.setup(self.settings)
models_paths = split_strip(self.options.models)
for path in models_paths:
model = resolve(path)
model_name = path.split('.')[-1]
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urlparse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
es.index(documents, chunk_size=chunk_size)
else:
es.index_missing(documents, chunk_size=chunk_size)
return 0
def run(self, quiet=False):
from nefertari.elasticsearch import ES
ES.setup(self.settings)
models_paths = split_strip(self.options.models)
for path in models_paths:
model = resolve(path)
model_name = path.split('.')[-1]
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urlparse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
es.index(documents, chunk_size=chunk_size)
else:
es.index_missing(documents, chunk_size=chunk_size)
return 0
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([[k, v[0]]
for k, v in urllib.parse.parse_qs(params).items()])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name,
index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(model_name))
es.index(documents)
else:
self.log.info(
'Indexing missing `{}` documents'.format(model_name))
es.index_missing_documents(documents)
return 0
def run(self):
ES.setup(self.settings)
model_names = split_strip(self.options.models)
for model_name in model_names:
self.log.info('Processing model `{}`'.format(model_name))
model = engine.get_document_cls(model_name)
params = self.options.params or ''
params = dict([
[k, v[0]] for k, v in urllib.parse.parse_qs(params).items()
])
params.setdefault('_limit', params.get('_limit', 10000))
chunk_size = self.options.chunk or params['_limit']
es = ES(source=model_name, index_name=self.options.index,
chunk_size=chunk_size)
query_set = model.get_collection(**params)
documents = to_dicts(query_set)
if self.options.force:
self.log.info('Recreating `{}` ES mapping'.format(model_name))
es.delete_mapping()
es.put_mapping(body=model.get_es_mapping())
self.log.info('Indexing all `{}` documents'.format(
model_name))
es.index(documents)
else:
self.log.info('Indexing missing `{}` documents'.format(
model_name))
es.index_missing_documents(documents)
return 0
def run(self):
ES.setup(self.settings)
if self.options.recreate:
self.recreate_index()
models = engine.get_document_classes()
model_names = [
name for name, model in models.items()
if getattr(model, '_index_enabled', False)]
else:
model_names = split_strip(self.options.models)
self.index_models(model_names)
def run(self):
ES.setup(self.settings)
if self.options.recreate:
self.recreate_index()
models = engine.get_document_classes()
model_names = [
name for name, model in models.items()
if getattr(model, '_index_enabled', False)
]
else:
model_names = split_strip(self.options.models)
self.index_models(model_names)
def main(argv=sys.argv):
global log
args = parse_args()
options = parse_vars(args.options)
settings = get_appsettings(args.ini, name="hathor", options=options)
setup_logging(args.ini)
log = logging.getLogger()
## Init underlaying FW -- Code inspire from nefertari
# see: https://github.com/ramses-tech/nefertari/blob/master/nefertari/scripts/es.py
mappings_setup = getattr(ES, '_mappings_setup', False)
try:
ES._mappings_setup = True
env = bootstrap("%s#hathor" % args.ini)
## Not sure if I need this, bootstrap should take care of this
config = Configurator(settings=settings)
config.include('nefertari.engine')
finally:
ES._mappings_setup = mappings_setup
registry = env['registry']
ES.setup(dictset(registry.settings))
# Recreate: drop index, get all available model names -- ignore -m from
# arguments list
if args.recreate and not args.list:
recreate_index()
args.models = available_models()
if args.list:
print("Available models:\n {}".format(", ".join(available_models())))
elif args.models:
model_names = args.models
if model_names:
av_models = available_models()
for elem in model_names:
if not elem in av_models:
raise (ValueError(
"model '{}' not available."
"Use '-l' to list available models.".format(elem)))
## still here we gonna re-index things
for model in model_names:
if args.delete_mapping:
recreate_mapping(model)
reindextask(model, boxsize=args.boxsize)
